diff options
33 files changed, 3264 insertions, 1067 deletions
@@ -86,3 +86,6 @@ Stanislaw Pitucha <stanislaw.pitucha@hpe.com> <stanislaw.pitucha@hp.com> Mahati Chamarthy <mahati.chamarthy@gmail.com> Peter Lisak <peter.lisak@firma.seznam.cz> Doug Hellmann <doug@doughellmann.com> <doug.hellmann@dreamhost.com> +Ondrej Novy <ondrej.novy@firma.seznam.cz> +James Nzomo <james@tdt.rocks> <kazikubwa@gmail.com> +Alessandro Pilotti <ap@pilotti.it> <apilotti@cloudbasesolutions.com> diff --git a/.manpages b/.manpages new file mode 100755 index 0000000..69fcfc7 --- /dev/null +++ b/.manpages @@ -0,0 +1,18 @@ +#!/bin/sh + +RET=0 +for MAN in doc/manpages/* ; do + OUTPUT=$(LC_ALL=en_US.UTF-8 MANROFFSEQ='' MANWIDTH=80 man --warnings -E UTF-8 -l \ + -Tutf8 -Z "$MAN" 2>&1 >/dev/null) + if [ -n "$OUTPUT" ] ; then + RET=1 + echo "$MAN:" + echo "$OUTPUT" + fi +done + +if [ "$RET" -eq "0" ] ; then + echo "All manpages are fine" +fi + +exit "$RET" @@ -10,11 +10,14 @@ Clint Byrum (clint@fewbar.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Mahati Chamarthy (mahati.chamarthy@gmail.com) +Chaozhe.Chen (chaozhe.chen@easystack.cn) Ray Chen (oldsharp@163.com) +Li Cheng (shcli@cn.ibm.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) Alistair Coles (alistair.coles@hpe.com) Ian Cordasco (ian.cordasco@rackspace.com) Nick Craig-Wood (nick@craig-wood.com) +Thiago da Silva (thiago@redhat.com) Sean Dague (sean@dague.net) Zack M. Davis (zdavis@swiftstack.com) John Dickinson (me@not.mn) @@ -38,7 +41,7 @@ Charles Hsu (charles0126@gmail.com) Kun Huang (gareth@unitedstack.com) Matthieu Huin (mhu@enovance.com) Andreas Jaeger (aj@suse.de) -OpenStack Jenkins (jenkins@openstack.org) +Jude Job (judeopenstack@gmail.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) Leah Klearman (lklrmn@gmail.com) Jaivish Kothari (jaivish.kothari@nectechnologies.in) @@ -52,6 +55,7 @@ Peter Lisak (peter.lisak@firma.seznam.cz) Feng Liu (mefengliu23@gmail.com) Jing Liuqing (jing.liuqing@99cloud.net) Hemanth Makkapati (hemanth.makkapati@mailtrust.com) +Pratik Mallya (pratik.mallya@gmail.com) Steve Martinelli (stevemar@ca.ibm.com) Juan J. Martinez (juan@memset.com) Donagh McCabe (donagh.mccabe@hpe.com) @@ -59,13 +63,14 @@ Ben McCann (ben@benmccann.com) Andy McCrae (andy.mccrae@gmail.com) Stuart McLaren (stuart.mclaren@hpe.com) Samuel Merritt (sam@swiftstack.com) +Min Min Ren (rminmin@cn.ibm.com) Jola Mirecka (jola.mirecka@hp.com) Hiroshi Miura (miurahr@nttdata.co.jp) Sam Morrison (sorrison@gmail.com) Dirk Mueller (dirk@dmllr.de) Zhenguo Niu (zhenguo@unitedstack.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) -Alessandro Pilotti (apilotti@cloudbasesolutions.com) +James Nzomo (james@tdt.rocks) Alessandro Pilotti (ap@pilotti.it) Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Dan Prince (dprince@redhat.com) @@ -77,6 +82,7 @@ Mark Seger (mark.seger@hpe.com) Chuck Short (chuck.short@canonical.com) David Shrewsbury (shrewsbury.dave@gmail.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) +Alexandra Settle (alexandra.settle@rackspace.com) Jeremy Stanley (fungi@yuggoth.org) Victor Stinner (victor.stinner@enovance.com) Jiří Suchomel (jsuchome@suse.cz) @@ -103,3 +109,6 @@ tanlin (lin.tan@intel.com) yangxurong (yangxurong@huawei.com) yuxcer (yuxcer@126.com) zhang-jinnan (ben.os@99cloud.net) +hgangwx (hgangwx@cn.ibm.com) +shu-mutou (shu-mutou@rf.jp.nec.com) +SaiKiran (saikiranveeravarapu@gmail.com) @@ -1,3 +1,45 @@ +3.0.0 +----- + +* Python 2.6 and Python 3.3 support has been removed. Currently + supported and tested versions of Python are Python 2.7 and Python 3.4. + +* Do not reveal sensitive headers in swiftclient log messages by default. + This is controlled by the client.logger_settings dictionary. Setting the + `redact_sensitive_headers` key to False prevents the information hiding. If + the value is True (the default), the `reveal_sensitive_prefix` controls + the maximum length of any sensitive header value logged. The default is + 16 to match the default in Swift. + +* Object downloads that fail partway through will now retry with a Range + request to read the rest of the object. + +* Object uploads will be retried if the source supports seek/tell or has a + reset() method. + +* Delete requests will use the cluster's bulk delete feature, if available, + for requests that would require a lot of individual deletes. + +* The delete CLI option now accepts a --prefix option to delete objects that + start with the given prefix (similar to the same-named option for list). + +* Add support for the auth-version to be specified using + --os-identity-api-version or OS_IDENTITY_API_VERSION + for compatibility with other openstack client command + line options. + +* --debug and --info command-line options now work anywhere in the command. + +* Objects can now be uploaded to pseudo-directories with the CLI. + +* Fixed an issue with uploading a large object that includes a unicode path. + +* swiftclient can now auth against Keystone using only a project (tenant) + and a token. This is useful when the client doesn't have access to the + password for a user but otherwise has been granted access. + +* Various other minor bug fixes and improvements. + 2.7.0 ----- diff --git a/doc/source/cli.rst b/doc/source/cli.rst new file mode 100644 index 0000000..12de02f --- /dev/null +++ b/doc/source/cli.rst @@ -0,0 +1,334 @@ +==== +CLI +==== + +The ``swift`` tool is a command line utility for communicating with an OpenStack +Object Storage (swift) environment. It allows one to perform several types of +operations. + +Authentication +~~~~~~~~~~~~~~ + +This section covers the options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below, but are just a subset of those that can be used +to successfully authenticate. These are the most common and recommended +combinations. + +You should obtain the details of your authentication version and credentials +from your storage provider. These details should make it clearer which of the +authentication sections below are most likely to allow you to connect to your +storage account. + +Keystone v3 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-name project1 --os-project-domain-name domain1 \ + --os-username user --os-user-domain-name domain1 \ + --os-password password list + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-id 0123456789abcdef0123456789abcdef \ + --os-user-id abcdef0123456789abcdef0123456789 \ + --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following combinations of environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=3 + OS_USERNAME=user + OS_USER_DOMAIN_NAME=domain1 + OS_PASSWORD=password + OS_PROJECT_NAME=project1 + OS_PROJECT_DOMAIN_NAME=domain1 + OS_AUTH_URL=https://api.example.com:5000/v3 + + ST_AUTH_VERSION=3 + OS_USER_ID=abcdef0123456789abcdef0123456789 + OS_PASSWORD=password + OS_PROJECT_ID=0123456789abcdef0123456789abcdef + OS_AUTH_URL=https://api.example.com:5000/v3 + +Keystone v2 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v2.0 \ + --os-tenant-name tenant \ + --os-username user --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=2.0 + OS_USERNAME=user + OS_PASSWORD=password + OS_TENANT_NAME=tenant + OS_AUTH_URL=https://api.example.com:5000/v2.0 + +Legacy auth systems +------------------- + +You can configure swift to work with any number of other authentication systems +that we will not cover in this document. If your storage provider is not using +Keystone to provide access tokens, please contact them for instructions on the +required options. It is likely that the options will need to be specified as +below: + +.. code-block:: bash + + swift -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key list + +Specifying the options above manually on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=1.0 + ST_AUTH=https://auth.api.rackspacecloud.com/v1.0 + ST_USER=user + ST_KEY=key + +It is also possible that you need to use a completely separate auth system, in which +case ``swiftclient`` cannot request a token for you. In this case you should make the +authentication request separately and access your storage using the token and +storage URL options shown below: + +.. code-block:: bash + + swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ + list + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + Leftover environment variables are a common source of confusion when + authorization fails. + +CLI commands +~~~~~~~~~~~~ + +Stat +---- + + ``stat [container [object]]`` + + Displays information for the account, container, or object depending on + the arguments given (if any). In verbose mode, the storage URL and the + authentication token are displayed as well. + +List +---- + + ``list [command-options] [container]`` + + Lists the containers for the account or the objects for a container. + The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only + list items beginning with that prefix. The ``-d <delimiter>`` or + ``--delimiter <delimiter>`` is an option (for container listings only) + that will roll up items with the given delimiter (see `OpenStack Swift + general documentation <http://docs.openstack.org/developer/swift/>` for + what this means). + + The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` + and ``ls -lh``, the latter providing sizes in human readable format + (For example: ``3K``, ``12M``, etc). The latter two switches use more + overhead to retrieve the displayed details, which is directly proportional + to the number of container or objects listed. + +Upload +------ + + ``upload [command-options] container file_or_directory [file_or_directory] [...]`` + + Uploads the files and directories specified by the remaining arguments to the + given container. The ``-c`` or ``--changed`` is an option that will only + upload files that have changed since the last upload. The + ``--object-name <object-name>`` is an option that will upload a file and + name object to ``<object-name>`` or upload a directory and use ``<object-name>`` + as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and + ``--leave-segments`` are options as well (see ``--help`` for more). + +Post +---- + + ``post [command-options] [container] [object]`` + + Updates meta information for the account, container, or object depending + on the arguments given. If the container is not found, the ``swiftclient`` + will create it automatically, but this is not true for accounts and + objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl + <read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options. + The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, + and is used to define the user metadata items to set in the form ``Name:Value``. + You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` + + For more information about ACL formats see the documentation: + `ACLs <http://docs.openstack.org/developer/swift/misc.html#acls/>`_. + +Download +-------- + + ``download [command-options] [container] [object] [object] [...]`` + + Downloads everything in the account (with ``--all``), or everything in a + container, or a list of objects depending on the arguments given. For a + single object download, you may use the ``-o <filename>`` or ``--output <filename>`` + option to redirect the output to a specific file or ``-`` to + redirect to stdout. You can specify optional headers with the repeatable + cURL-like option ``-H [--header <name:value>]``. + +Delete +------ + + ``delete [command-options] [container] [object] [object] [...]`` + + Deletes everything in the account (with ``--all``), or everything in a + container, or a list of objects depending on the arguments given. Segments + of manifest objects will be deleted as well, unless you specify the + ``--leave-segments`` option. + +Capabilities +------------ + + ``capabilities [proxy-url]`` + + Displays cluster capabilities. The output includes the list of the + activated Swift middlewares as well as relevant options for each ones. + Additionally the command displays relevant options for the Swift core. If + the ``proxy-url`` option is not provided, the storage URL retrieved after + authentication is used as ``proxy-url``. + + +Examples +~~~~~~~~ + +In this section we present some example usage of the ``swift`` CLI. To keep the +examples as short as possible, these examples assume that the relevant authentication +options have been set using environment variables. You can obtain the full list of +commands and options available in the ``swift`` CLI by executing the following: + +.. code-block:: bash + + > swift --help + > swift <command> --help + +Simple examples +--------------- + +List the existing swift containers: + +.. code-block:: bash + + > swift list + + container_1 + +Create a new container: + +.. code-block:: bash + + > swift post TestContainer + +Upload an object into a container: + +.. code-block:: bash + + > swift upload TestContainer testSwift.txt + + testSwift.txt + +List the contents of a container: + +.. code-block:: bash + + > swift list TestContainer + + testSwift.txt + +Download an object from a container: + +.. code-block:: bash + + > swift download TestContainer testSwift.txt + + testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In the case that you provide the complete path of the file, that complete + path will be the name of the uploaded object. + +For example: + +.. code-block:: bash + + > swift upload TestContainer /home/swift/testSwift/testSwift.txt + + home/swift/testSwift/testSwift.txt + + > swift list TestContainer + + home/swift/testSwift/testSwift.txt + +More complex examples +--------------------- + +Swift has a single object size limit of 5GiB. In order to upload files larger +than this, we must create a large object that consists of smaller segments. +The example below shows how to upload a large video file as a static large +object in 1GiB segments: + +.. code-block:: bash + + > swift upload videos --use-slo --segment-size 1G myvideo.mp4 + + myvideo.mp4 segment 8 + myvideo.mp4 segment 4 + myvideo.mp4 segment 2 + myvideo.mp4 segment 7 + myvideo.mp4 segment 0 + myvideo.mp4 segment 1 + myvideo.mp4 segment 3 + myvideo.mp4 segment 6 + myvideo.mp4 segment 5 + myvideo.mp4 + +This command will upload segments to a container named ``videos_segments``, and +create a manifest file describing the entire object in the ``videos`` container. +For more information on large objects, see the documentation `here +<http://docs.openstack.org/developer/swift/overview_large_objects.html>`_. + +.. code-block:: bash + + > swift list videos + + myvideo.mp4 + + > swift list videos_segments + + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 diff --git a/doc/source/client-api.rst b/doc/source/client-api.rst new file mode 100644 index 0000000..5677f70 --- /dev/null +++ b/doc/source/client-api.rst @@ -0,0 +1,177 @@ +============================== +The swiftclient.Connection API +============================== + +A low level API that provides methods for authentication and methods that +correspond to the individual REST API calls described in the swift +documentation. + +For usage details see the client docs: :mod:`swiftclient.client`. + +Authentication +-------------- + +This section covers the various combinations of kwargs required when creating +and instance of the ``Connection`` object for communicating with a swift +object store. The combinations of options required for each authentication +version are detailed below, but are +just a subset of those that can be used to successfully authenticate. These +are the most common and recommended combinations. + +Keystone v3 +~~~~~~~~~~~ + +.. code-block:: python + + _authurl = 'http://127.0.0.1:5000/v3/' + _auth_version = '3' + _user = 'tester' + _key = 'testing' + _os_options = { + 'user_domain_name': 'Default', + 'project_domain_name': 'Default', + 'project_name': 'Default' + } + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + os_options=_os_options, + auth_version=_auth_version + ) + +.. code-block:: python + + _authurl = 'http://127.0.0.1:5000/v3/' + _auth_version = '3' + _user = 'tester' + _key = 'testing' + _os_options = { + 'user_domain_id': 'Default', + 'project_domain_id': 'Default', + 'project_id': 'Default' + } + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + os_options=_os_options, + auth_version=_auth_version + ) + +Keystone v2 +~~~~~~~~~~~ + +.. code-block:: python + + _authurl = 'http://127.0.0.1:5000/v2.0/' + _auth_version = '2' + _user = 'tester' + _key = 'testing' + _tenant_name = 'test' + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + tenant_name=_tenant_name, + auth_version=_auth_version + ) + +Legacy Auth +~~~~~~~~~~~ + +.. code-block:: python + + _authurl = 'http://127.0.0.1:8080/' + _auth_version = '1' + _user = 'tester' + _key = 'testing' + _tenant_name = 'test' + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + tenant_name=_tenant_name, + auth_version=_auth_version + ) + +Examples +-------- + +In this section we present some simple code examples that demonstrate the usage +of the ``Connection`` API. You can find full details of the options and methods +available to the ``Connection`` API in the docstring generated documentation: +:mod:`swiftclient.client`. + +List the available containers: + +.. code-block:: python + + resp_headers, containers = conn.get_account() + print("Response headers: %s" % resp_headers) + for container in containers: + print(container) + +Create a new container: + +.. code-block:: python + + container = 'new-container' + conn.put_container(container) + resp_headers, containers = conn.get_account() + if container in containers: + print("The container was created") + +Create a new object with the contents of a local text file: + +.. code-block:: python + + container = 'new-container' + with open('local.txt', 'r') as local: + conn.put_object( + container, + 'local_object.txt', + contents=local, + content_type='text/plain' + ) + +Confirm presence of the object: + +.. code-block:: python + + obj = 'local_object.txt' + container = 'new-container' + try: + resp_headers = conn.head_object(container, obj) + print('The object was successfully created') + except ClientException as e: + if e.http_status = '404': + print('The object was not found') + else: + print('An error occurred checking for the existence of the object') + +Download the created object: + +.. code-block:: python + + obj = 'local_object.txt' + container = 'new-container' + resp_headers, obj_contents = conn.get_object(container, obj) + with open('local_copy.txt', 'w') as local: + local.write(obj_contents) + +Delete the created object: + +.. code-block:: python + + obj = 'local_object.txt' + container = 'new-container' + try: + conn.delete_object(container, obj) + print("Successfully deleted the object") + except ClientException as e: + print("Failed to delete the object with error: %s" % e) diff --git a/doc/source/index.rst b/doc/source/index.rst index 3b8535a..f123b7b 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,16 +1,28 @@ +====================================== Welcome to the python-swiftclient Docs -************************************** +====================================== + +Introduction +~~~~~~~~~~~~ + +.. toctree:: + :maxdepth: 2 + + introduction Developer Documentation -======================= +~~~~~~~~~~~~~~~~~~~~~~~ .. toctree:: :maxdepth: 2 - apis + cli + service-api + client-api + Code-Generated Documentation -============================ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. toctree:: :maxdepth: 2 @@ -18,14 +30,14 @@ Code-Generated Documentation swiftclient Indices and tables -================== +~~~~~~~~~~~~~~~~~~ * :ref:`genindex` * :ref:`modindex` * :ref:`search` License -======= +~~~~~~~ Copyright 2013 OpenStack, LLC. diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst new file mode 100644 index 0000000..926b1b9 --- /dev/null +++ b/doc/source/introduction.rst @@ -0,0 +1,94 @@ +============ +Introduction +============ + +Where to Start? +~~~~~~~~~~~~~~~ + +The ``python-swiftclient`` project comprises a command line tool and two +separate APIs for accessing swift programmatically. Choosing the most +appropriate method for a given use case is the first problem a user needs to +solve. + +Use Cases +--------- + +Alongside the command line tool, the ``python-swiftclient`` includes two +levels of API: + + * A low level client API that provides simple Python wrappers around the + various authentication mechanisms and the individual HTTP requests. + * A high level service API that provides methods for performing common + operations in parallel on a thread pool. + +Example use cases: + + * Uploading and retrieving data + Use the command line tool if you are simply uploading and downloading + files and directories to and from your filesystem. The command line tool + can be integrated into a shell script to automate tasks. + + * Integrating into an automated Python workflow + Use the ``SwiftService`` API to perform operations offered by the CLI + if your use case requires integration with a Python-based workflow. + This method offers greater control and flexibility over individual object + operations, such as the metadata set on each object. The ``SwiftService`` + class provides methods to perform multiple sets of operations against a + swift object store using a configurable shared thread pool. A single + instance of the ``SwiftService`` class can be shared between multiple + threads in your own code. + + * Developing an application in Python to access a swift object store + Use the ``SwiftService`` API to develop Python applications that use + swift to store and retrieve objects. A ``SwiftService`` instance provides + a configurable thread pool for performing all operations supported by the + CLI. + + * Fine-grained control over threading or the requests being performed + Use the ``Connection`` API if your use case requires fine grained control + over advanced features or you wish to use your own existing threading + model. Examples of advanced features requiring the use of the + ``Connection`` API include creating an SLO manifest that references + already existing objects, or fine grained control over the query strings + supplied with each HTTP request. + +Important considerations +~~~~~~~~~~~~~~~~~~~~~~~~ + +This section covers some important considerations, helpful hints, and things to +avoid when integrating an object store into your workflow. + +An object store is not a filesystem +----------------------------------- + +It cannot be stressed enough that your usage of the object store should reflect +the proper use case, and not treat the storage like a traditional filesystem. +There are two main restrictions to bear in mind when designing an application +that uses an object store: + + * You cannot rename objects. Due to fact that the name of an object is one + of the factors that determines where the object and its replicas are stored, + renaming would require multiple copies of the data to be moved between + physical storage devices. If you want to rename an object you must upload + to the new location, or make a server side copy request to the new location, + and then delete the original. + + * You cannot modify objects. Objects are stored in multiple locations and + are checked for integrity based on the MD5 sum calculated during + upload. In order to modify the contents of an object, the entire desired + contents must be re-uploaded. In certain special cases it is possible to + work around this restriction using large objects, but no general + file-like access is available to modify a stored object. + +Objects cannot be locked +------------------------ + +There is no mechanism to perform a combination of reading the +data/metadata from an object and writing an update to that data/metadata in an +atomic way. Any user with access to a container could update the contents or +metadata associated with an object at any time. + +Workflows that assume that no updates have been made since the last read of an +object should be discouraged. Enabling a workflow of this type requires an +external object locking mechanism and/or cooperation between all clients +accessing the data. diff --git a/doc/source/apis.rst b/doc/source/service-api.rst index 1a8e8f7..7d65fd1 100644 --- a/doc/source/apis.rst +++ b/doc/source/service-api.rst @@ -1,60 +1,93 @@ -============ -Introduction -============ - -The python-swiftclient includes two levels of API; a low level client API that -provides simple python wrappers around the various authentication mechanisms -and the individual HTTP requests, and a high level service API that provides -methods for performing common operations in parallel on a thread pool. - -This document aims to provide guidance for choosing between these APIs and -examples of usage for the service API. - ------------------------- -Important Considerations ------------------------- - -This section covers some important considerations, helpful hints, and things -to avoid when integrating an object store into your workflow. - -An Object Store is not a filesystem ------------------------------------ - -It cannot be stressed enough that your usage of the object store should reflect -the proper use case, and not treat the storage like a filesystem. There are 2 -main restrictions to bear in mind here when designing your use of the object -store: - - * Objects cannot be renamed due to the way in which objects are stored and - references by the object store. This usually requires multiple copies of - the data to be moved between physical storage devices. - As a result, a move operation is not provided. If the user wants to move an - object they must re-upload to the new location and delete the - original. - * Objects cannot be modified. Objects are stored in multiple locations and are - checked for integrity based on the ``MD5 sum`` calculated during upload. - Object creation is a 1-shot event, and in order to modify the contents of an - object the entire new contents must be re-uploaded. In certain special cases - it is possible to work around this restriction using large objects, but no - general file-like access is available to modify a stored object. - ------------------------------- -The swiftclient.Connection API ------------------------------- - -A low level API that provides methods for authentication and methods that -correspond to the individual REST API calls described in the swift -documentation. - -For usage details see the client docs: :mod:`swiftclient.client`. - --------------------------------- +================================ The swiftclient.SwiftService API --------------------------------- +================================ -A higher level API aimed at allowing developers an easy way to perform multiple -operations asynchronously using a configurable thread pool. Docs for each -service method call can be found here: :mod:`swiftclient.service`. +A higher-level API aimed at allowing developers an easy way to perform multiple +operations asynchronously using a configurable thread pool. Documentation for +each service method call can be found here: :mod:`swiftclient.service`. + +Authentication +-------------- + +This section covers the various options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below. Once again, these are just a subset of those that +can be used to successfully authenticate, but they are the most common and +recommended. + +The relevant authentication options are presented as python dictionaries that +should be added to any other options you are supplying to your ``SwiftService`` +instance. As indicated in the python code, you can also set these options as +environment variables that will be loaded automatically if the relevant option +is not specified. + +The ``SwiftService`` authentication attempts to automatically select +the auth version based on the combination of options specified, but +supplying options from multiple different auth versions can cause unexpected +behaviour. + + .. note:: + + Leftover environment variables are a common source of confusion when + authorization fails. + +Keystone V3 +~~~~~~~~~~~ + +.. code-block:: python + + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '3' + "os_username": environ.get('OS_USERNAME'), + "os_password": environ.get('OS_PASSWORD'), + "os_project_name": environ.get('OS_PROJECT_NAME'), + "os_project_domain_name": environ.get('OS_PROJECT_DOMAIN_NAME'), + "os_auth_url": environ.get('OS_AUTH_URL'), + ... + } + +.. code-block:: python + + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '3' + "os_username": environ.get('OS_USERNAME'), + "os_password": environ.get('OS_PASSWORD'), + "os_project_id": environ.get('OS_PROJECT_ID'), + "os_project_domain_id": environ.get('OS_PROJECT_DOMAIN_ID'), + "os_auth_url": environ.get('OS_AUTH_URL'), + ... + } + +Keystone V2 +~~~~~~~~~~~ + +.. code-block:: python + + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '2.0' + "os_username": environ.get('OS_USERNAME'), + "os_password": environ.get('OS_PASSWORD'), + "os_tenant_name": environ.get('OS_TENANT_NAME'), + "os_auth_url": environ.get('OS_AUTH_URL'), + ... + } + +Legacy Auth +~~~~~~~~~~~ + +.. code-block:: python + + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '1.0' + "auth": environ.get('ST_AUTH'), + "user": environ.get('ST_USER'), + "key": environ.get('ST_KEY'), + ... + } Configuration ------------- @@ -189,51 +222,8 @@ source code for ``python-swiftclient``. Each ``SwiftService`` method also allows for an optional dictionary to override those specified at init time, and the appropriate docstrings show which options modify each method's behaviour. -Authentication --------------- - -This section covers the various options for authenticating with a swift -object store. The combinations of options required for each authentication -version are detailed below. - -Version 1.0 Auth -~~~~~~~~~~~~~~~~ - - ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` - - ``auth``: ``environ.get('ST_AUTH')`` - - ``user``: ``environ.get('ST_USER')`` - - ``key``: ``environ.get('ST_KEY')`` - - -Version 2.0 & 3.0 Auth -~~~~~~~~~~~~~~~~~~~~~~ - - ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` - - ``os_username``: ``environ.get('OS_USERNAME')`` - - ``os_password``: ``environ.get('OS_PASSWORD')`` - - ``os_tenant_name``: ``environ.get('OS_TENANT_NAME')`` - - ``os_auth_url``: ``environ.get('OS_AUTH_URL')`` - -As is evident from the default values, if these options are not set explicitly -in the options dictionary, then they will default to the values of the given -environment variables. The ``SwiftService`` authentication automatically selects -the auth version based on the combination of options specified, but -having options from different auth versions can cause unexpected behaviour. - - .. note:: - - Leftover environment variables are a common source of confusion when - authorization fails. - -Operation Return Values ------------------------ +Available Operations +-------------------- Each operation provided by the service API may raise a ``SwiftError`` or ``ClientException`` for any call that fails completely (or a call which @@ -291,7 +281,7 @@ All the possible ``action`` values are detailed below: ] Stat ----- +~~~~ Stat can be called against an account, a container, or a list of objects to get account stats, container stats or information about the given objects. In @@ -368,35 +358,17 @@ operation was not successful, and will include the keys below: } Example -~~~~~~~ +^^^^^^^ The code below demonstrates the use of ``stat`` to retrieve the headers for a given list of objects in a container using 20 threads. The code creates a -mapping from object name to headers. +mapping from object name to headers which is then pretty printed to the log. -.. code-block:: python - - import logging - - from swiftclient.service import SwiftService - - logger = logging.getLogger() - _opts = {'object_dd_threads': 20} - with SwiftService(options=_opts) as swift: - container = 'container1' - objects = [ 'object_%s' % n for n in range(0,100) ] - header_data = {} - stats_it = swift.stat(container=container, objects=objects) - for stat_res in stats_it: - if stat_res['success']: - header_data[stat_res['object']] = stat_res['headers'] - else: - logger.error( - 'Failed to retrieve stats for %s' % stat_res['object'] - ) +.. literalinclude:: ../../examples/stat.py + :language: python List ----- +~~~~ List can be called against an account or a container to retrieve the containers or objects contained within them. Each call returns an iterator that returns @@ -453,55 +425,38 @@ dictionary as described below: } Example -~~~~~~~ +^^^^^^^ The code below demonstrates the use of ``list`` to list all items in a container that are over 10MiB in size: -.. code-block:: python - - container = 'example_container' - minimum_size = 10*1024**2 - with SwiftService() as swift: - try: - stats_parts_gen = swift.list(container=container) - for stats in stats_parts_gen: - if stats["success"]: - for item in stats["listing"]: - i_size = int(item["bytes"]) - if i_size > minimum_size: - i_name = item["name"] - i_etag = item["hash"] - print( - "%s [size: %s] [etag: %s]" % - (i_name, i_size, i_etag) - ) - else: - raise stats["error"] - except SwiftError as e: - output_manager.error(e.value) +.. literalinclude:: ../../examples/list.py + :language: python Post ----- +~~~~ Post can be called against an account, container or list of objects in order to -update the metadata attached to the given items. Each element of the object list -may be a plain string of the object name, or a ``SwiftPostObject`` that -allows finer control over the options applied to each of the individual post -operations. In the first two cases a single dictionary is returned containing the -results of the operation, and in the case of a list of objects being supplied, -an iterator over the results generated for each object post is returned. If the -given container or account does not exist, the ``post`` method will raise a -``SwiftError``. - -When a string is given for the object name, the options - -Successful metadata update results are dictionaries as described below: +update the metadata attached to the given items. In the first two cases a single +dictionary is returned containing the results of the operation, and in the case +of a list of objects being supplied, an iterator over the results generated for +each object post is returned. + +Each element of the object list may be a plain string of the object name, or a +``SwiftPostObject`` that allows finer control over the options and metadata +applied to each of the individual post operations. When a string is given for +the object name, the options and metadata applied are a combination of those +supplied to the call to ``post()`` and the defaults of the ``SwiftService`` +object. + +If the given container or account does not exist, the ``post`` method will +raise a ``SwiftError``. Successful metadata update results are dictionaries as +described below: .. code-block:: python { - 'action': <'post_account'|<'post_container'>|'post_object'>, + 'action': <'post_account'|'post_container'|'post_object'>, 'success': True, 'container': <container>, 'object': <object>, @@ -516,28 +471,84 @@ Successful metadata update results are dictionaries as described below: key-value pairs must be specified. Example -~~~~~~~ +^^^^^^^ -.. Do we want to hide this section until it is complete? +The code below demonstrates the use of ``post`` to set an archive folder in a +given container to expire after a 24 hour delay: -TBD +.. literalinclude:: ../../examples/post.py + :language: python Download --------- +~~~~~~~~ + +Download can be called against an entire account, a single container, or a list +of objects in a given container. Each element of the object list is a string +detailing the full name of an object to download. + +In order to download the full contents of an entire account, you must set the +value of ``yes_all`` to ``True`` in the ``options`` dictionary supplied to +either the ``SwiftService`` instance or the call to ``download``. + +If the given container or account does not exist, the ``download`` method will +raise a ``SwiftError``, otherwise an iterator over the results generated for +each object download is returned. -.. Do we want to hide this section until it is complete? +See :mod:`swiftclient.service.SwiftService.download` for docs generated from the +method docstring. + +For each successfully downloaded object, the results returned by the iterator +will be a dictionary as described below (results are not returned for completed +container or object segment downloads): + +.. code-block:: python -TBD + { + 'action': 'download_object', + 'container': <container>, + 'object': <object name>, + 'success': True, + 'path': <local path to downloaded object>, + 'pseudodir': <if true, the download created an empty directory>, + 'start_time': <time download started>, + 'end_time': <time download completed>, + 'headers_receipt': <time the headers from the object were retrieved>, + 'auth_end_time': <time authentication completed>, + 'read_length': <bytes_read>, + 'attempts': <attempt count>, + 'response_dict': <HTTP response details> + } + +Any failure uploading an object will return a failure dictionary as described +below: + +.. code-block:: python + + { + 'action': 'download_object', + 'container': <container>, + 'object': <object name>, + 'success': False, + 'path': <local path of the failed download>, + 'pseudodir': <if true, the failed download was an empty directory>, + 'attempts': <attempt count>, + 'error': <error>, + 'traceback': <trace>, + 'error_timestamp': <timestamp>, + 'response_dict': <HTTP response details> + } Example -~~~~~~~ +^^^^^^^ -.. Do we want to hide this section until it is complete? +The code below demonstrates the use of ``download`` to download all PNG images +from a dated archive folder in a given container: -TBD +.. literalinclude:: ../../examples/download.py + :language: python Upload ------- +~~~~~~ Upload is always called against an account and container and with a list of objects to upload. Each element of the object list may be a plain string @@ -550,7 +561,7 @@ the upload are those supplied to the call to ``upload``. Constructing a ``SwiftUploadObject`` allows the user to supply an object name for the uploaded file, and modify the options used by ``upload`` at the -granularity of invidivual files. +granularity of individual files. If the given container or account does not exist, the ``upload`` method will raise a ``SwiftError``, otherwise an iterator over the results generated for @@ -622,98 +633,195 @@ below: } Example -~~~~~~~ +^^^^^^^ The code below demonstrates the use of ``upload`` to upload all files and -folders in ``/tmp``, and renaming each object by replacing ``/tmp`` in the -object or directory marker names with ``temporary-objects``: +folders in a given directory, and rename each object by replacing the root +directory name with 'my-<d>-objects', where <d> is the name of the uploaded +directory: + +.. literalinclude:: ../../examples/upload.py + :language: python + +Delete +~~~~~~ + +Delete can be called against an account or a container to remove the containers +or objects contained within them. Each call to ``delete`` returns an iterator +over results of each resulting sub-request. + +If the number of requested delete operations is large and the target swift +cluster is running the bulk middleware, the call to ``SwiftService.delete`` will +make use of bulk operations and the returned result iterator will return +``bulk_delete`` results rather than individual ``delete_object``, +``delete_container`` or ``delete_segment`` results. + +See :mod:`swiftclient.service.SwiftService.delete` for docs generated from the +method docstring. + +For each successfully deleted container, object or segment, the results returned +by the iterator will be a dictionary as described below: .. code-block:: python - _opts['object_uu_threads'] = 20 - with SwiftService(options=_opts) as swift, OutputManager() as out_manager: - try: - # Collect all the files and folders in '/tmp' - objs = [] - dir_markers = [] - dir = '/tmp': - for (_dir, _ds, _fs) in walk(f): - if not (_ds + _fs): - dir_markers.append(_dir) - else: - objs.extend([join(_dir, _f) for _f in _fs]) - - # Now that we've collected all the required files and dir markers - # build the ``SwiftUploadObject``s for the call to upload - objs = [ - SwiftUploadObject( - o, object_name=o.replace( - '/tmp', 'temporary-objects', 1 - ) - ) for o in objs - ] - dir_markers = [ - SwiftUploadObject( - None, object_name=d.replace( - '/tmp', 'temporary-objects', 1 - ), options={'dir_marker': True} - ) for d in dir_markers - ] - - # Schedule uploads on the SwiftService thread pool and iterate - # over the results - for r in swift.upload(container, objs + dir_markers): - if r['success']: - if 'object' in r: - out_manager.print_msg(r['object']) - elif 'for_object' in r: - out_manager.print_msg( - '%s segment %s' % (r['for_object'], - r['segment_index']) - ) - else: - error = r['error'] - if r['action'] == "create_container": - out_manager.warning( - 'Warning: failed to create container ' - "'%s'%s", container, msg - ) - elif r['action'] == "upload_object": - out_manager.error( - "Failed to upload object %s to container %s: %s" % - (container, r['object'], error) - ) - else: - out_manager.error("%s" % error) - - except SwiftError as e: - out_manager.error(e.value) + { + 'action': <'delete_object'|'delete_segment'>, + 'container': <container>, + 'object': <object name>, + 'success': True, + 'attempts': <attempt count>, + 'response_dict': <HTTP response details> + } -Delete ------- + { + 'action': 'delete_container', + 'container': <container>, + 'success': True, + 'response_dict': <HTTP response details>, + 'attempts': <attempt count> + } -.. Do we want to hide this section until it is complete? + { + 'action': 'bulk_delete', + 'container': <container>, + 'objects': <[objects]>, + 'success': True, + 'attempts': <attempt count>, + 'response_dict': <HTTP response details> + } -TBD +Any failure in a delete operation will return a failure dictionary as described +below: + +.. code-block:: python + + { + 'action': ('delete_object'|'delete_segment'), + 'container': <container>, + 'object': <object name>, + 'success': False, + 'attempts': <attempt count>, + 'error': <error>, + 'traceback': <trace>, + 'error_timestamp': <timestamp>, + 'response_dict': <HTTP response details> + } + + { + 'action': 'delete_container', + 'container': <container>, + 'success': False, + 'error': <error>, + 'traceback': <trace>, + 'error_timestamp': <timestamp>, + 'response_dict': <HTTP response details>, + 'attempts': <attempt count> + } + + { + 'action': 'bulk_delete', + 'container': <container>, + 'objects': <[objects]>, + 'success': False, + 'attempts': <attempt count>, + 'error': <error>, + 'traceback': <trace>, + 'error_timestamp': <timestamp>, + 'response_dict': <HTTP response details> + } Example -~~~~~~~ +^^^^^^^ -.. Do we want to hide this section until it is complete? +The code below demonstrates the use of ``delete`` to remove a given list of +objects from a specified container. As the objects are deleted the transaction +id of the relevant request is printed along with the object name and number +of attempts required. By printing the transaction id, the printed operations +can be easily linked to events in the swift server logs: -TBD +.. literalinclude:: ../../examples/delete.py + :language: python Capabilities ------------- +~~~~~~~~~~~~ -.. Do we want to hide this section until it is complete? +Capabilities can be called against an account or a particular proxy URL in +order to determine the capabilities of the swift cluster. These capabilities +include details about configuration options and the middlewares that are +installed in the proxy pipeline. -TBD +See :mod:`swiftclient.service.SwiftService.capabilities` for docs generated from +the method docstring. -Example -~~~~~~~ +For each successful call to list capabilities, a result dictionary will be +returned with the contents described below: -.. Do we want to hide this section until it is complete? + { + 'action': 'capabilities', + 'timestamp': <time of the call>, + 'success': True, + 'capabilities': <dictionary containing capability details> + } + +The contents of the capabilities dictionary contain the core swift capabilities +under the key ``swift``, all other keys show the configuration options for +additional middlewares deployed in the proxy pipeline. An example capabilities +dictionary is given below: + +.. code-block:: python + + { + 'account_quotas': {}, + 'bulk_delete': { + 'max_deletes_per_request': 10000, + 'max_failed_deletes': 1000 + }, + 'bulk_upload': { + 'max_containers_per_extraction': 10000, + 'max_failed_extractions': 1000 + }, + 'container_quotas': {}, + 'container_sync': {'realms': {}}, + 'formpost': {}, + 'keystoneauth': {}, + 'slo': { + 'max_manifest_segments': 1000, + 'max_manifest_size': 2097152, + 'min_segment_size': 1048576 + }, + 'swift': { + 'account_autocreate': True, + 'account_listing_limit': 10000, + 'allow_account_management': True, + 'container_listing_limit': 10000, + 'extra_header_count': 0, + 'max_account_name_length': 256, + 'max_container_name_length': 256, + 'max_file_size': 5368709122, + 'max_header_size': 8192, + 'max_meta_count': 90, + 'max_meta_name_length': 128, + 'max_meta_overall_size': 4096, + 'max_meta_value_length': 256, + 'max_object_name_length': 1024, + 'policies': [ + {'default': True, 'name': 'Policy-0'} + ], + 'strict_cors_mode': False, + 'version': '2.2.2' + }, + 'tempurl': { + 'methods': ['GET', 'HEAD', 'PUT'] + } + } + +Example +^^^^^^^ -TBD +The code below demonstrates the us of ``capabilities`` to determine if the +Swift cluster supports static large objects, and if so, the maximum number of +segments that can be described in a single manifest file, along with the +size restrictions on those objects: +.. literalinclude:: ../../examples/capabilities.py + :language: python diff --git a/examples/capabilities.py b/examples/capabilities.py new file mode 100644 index 0000000..024ad6f --- /dev/null +++ b/examples/capabilities.py @@ -0,0 +1,20 @@ +import logging + +from swiftclient.exceptions import ClientException +from swiftclient.service import SwiftService + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +with SwiftService() as swift: + try: + capabilities_result = swift.capabilities() + capabilities = capabilities_result['capabilities'] + if 'slo' in capabilities: + print('SLO is supported') + else: + print('SLO is not supported') + except ClientException as e: + logger.error(e.value) diff --git a/examples/delete.py b/examples/delete.py new file mode 100644 index 0000000..6979d9e --- /dev/null +++ b/examples/delete.py @@ -0,0 +1,34 @@ +import logging + +from swiftclient.service import SwiftService +from sys import argv + + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +_opts = {'object_dd_threads': 20} +container = argv[1] +objects = argv[2:] +with SwiftService(options=_opts) as swift: + del_iter = swift.delete(container=container, objects=objects) + for del_res in del_iter: + c = del_res.get('container', '') + o = del_res.get('object', '') + a = del_res.get('attempts') + if del_res['success'] and not del_res['action'] == 'bulk_delete': + rd = del_res.get('response_dict') + if rd is not None: + t = dict(rd.get('headers', {})) + if t: + print( + 'Successfully deleted {0}/{1} in {2} attempts ' + '(transaction id: {3})'.format(c, o, a, t) + ) + else: + print( + 'Successfully deleted {0}/{1} in {2} ' + 'attempts'.format(c, o, a) + ) diff --git a/examples/download.py b/examples/download.py new file mode 100644 index 0000000..5e3ebd0 --- /dev/null +++ b/examples/download.py @@ -0,0 +1,37 @@ +import logging + +from swiftclient.service import SwiftService, SwiftError +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +def is_png(obj): + return ( + obj["name"].lower().endswith('.png') or + obj["content_type"] == 'image/png' + ) + +container = argv[1] +with SwiftService() as swift: + try: + list_options = {"prefix": "archive_2016-01-01/"} + list_parts_gen = swift.list(container=container) + for page in list_parts_gen: + if page["success"]: + objects = [ + obj["name"] for obj in page["listing"] if is_png(obj) + ] + for down_res in swift.download( + container=container, + objects=objects): + if down_res['success']: + print("'%s' downloaded" % down_res['object']) + else: + print("'%s' download failed" % down_res['object']) + else: + raise page["error"] + except SwiftError as e: + logger.error(e.value) diff --git a/examples/list.py b/examples/list.py new file mode 100644 index 0000000..4b909d5 --- /dev/null +++ b/examples/list.py @@ -0,0 +1,32 @@ +import logging + +from swiftclient.service import SwiftService, SwiftError +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +container = argv[1] +minimum_size = 10*1024**2 +with SwiftService() as swift: + try: + list_parts_gen = swift.list(container=container) + for page in list_parts_gen: + if page["success"]: + for item in page["listing"]: + + i_size = int(item["bytes"]) + if i_size > minimum_size: + i_name = item["name"] + i_etag = item["hash"] + print( + "%s [size: %s] [etag: %s]" % + (i_name, i_size, i_etag) + ) + else: + raise page["error"] + + except SwiftError as e: + logger.error(e.value) diff --git a/examples/post.py b/examples/post.py new file mode 100644 index 0000000..c734543 --- /dev/null +++ b/examples/post.py @@ -0,0 +1,31 @@ +import logging + +from swiftclient.service import SwiftService, SwiftError +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +container = argv[1] +with SwiftService() as swift: + try: + list_options = {"prefix": "archive_2016-01-01/"} + list_parts_gen = swift.list(container=container) + for page in list_parts_gen: + if page["success"]: + objects = [obj["name"] for obj in page["listing"]] + post_options = {"header": "X-Delete-After:86400"} + for post_res in swift.post( + container=container, + objects=objects, + options=post_options): + if post_res['success']: + print("Object '%s' POST success" % post_res['object']) + else: + print("Object '%s' POST failed" % post_res['object']) + else: + raise page["error"] + except SwiftError as e: + logger.error(e.value) diff --git a/examples/stat.py b/examples/stat.py new file mode 100644 index 0000000..0905d1b --- /dev/null +++ b/examples/stat.py @@ -0,0 +1,25 @@ +import logging +import pprint + +from swiftclient.service import SwiftService +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +_opts = {'object_dd_threads': 20} +with SwiftService(options=_opts) as swift: + container = argv[1] + objects = argv[2:] + header_data = {} + stats_it = swift.stat(container=container, objects=objects) + for stat_res in stats_it: + if stat_res['success']: + header_data[stat_res['object']] = stat_res['headers'] + else: + logger.error( + 'Failed to retrieve stats for %s' % stat_res['object'] + ) + pprint.pprint(header_data) diff --git a/examples/upload.py b/examples/upload.py new file mode 100644 index 0000000..1b1e349 --- /dev/null +++ b/examples/upload.py @@ -0,0 +1,71 @@ +import logging + +from os.path import join, walk +from swiftclient.multithreading import OutputManager +from swiftclient.service import SwiftError, SwiftService, SwiftUploadObject +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +_opts = {'object_uu_threads': 20} +dir = argv[1] +container = argv[2] +with SwiftService(options=_opts) as swift, OutputManager() as out_manager: + try: + # Collect all the files and folders in the given directory + objs = [] + dir_markers = [] + for (_dir, _ds, _fs) in walk(dir): + if not (_ds + _fs): + dir_markers.append(_dir) + else: + objs.extend([join(_dir, _f) for _f in _fs]) + + # Now that we've collected all the required files and dir markers + # build the ``SwiftUploadObject``s for the call to upload + objs = [ + SwiftUploadObject( + o, object_name=o.replace( + dir, 'my-%s-objects' % dir, 1 + ) + ) for o in objs + ] + dir_markers = [ + SwiftUploadObject( + None, object_name=d.replace( + dir, 'my-%s-objects' % dir, 1 + ), options={'dir_marker': True} + ) for d in dir_markers + ] + + # Schedule uploads on the SwiftService thread pool and iterate + # over the results + for r in swift.upload(container, objs + dir_markers): + if r['success']: + if 'object' in r: + print(r['object']) + elif 'for_object' in r: + print( + '%s segment %s' % (r['for_object'], + r['segment_index']) + ) + else: + error = r['error'] + if r['action'] == "create_container": + logger.warning( + 'Warning: failed to create container ' + "'%s'%s", container, error + ) + elif r['action'] == "upload_object": + logger.error( + "Failed to upload object %s to container %s: %s" % + (container, r['object'], error) + ) + else: + logger.error("%s" % error) + + except SwiftError as e: + logger.error(e.value) @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://www.openstack.org/ +home-page = http://docs.openstack.org/developer/python-swiftclient classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -17,7 +17,6 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 @@ -33,6 +32,10 @@ scripts = data_files = share/man/man1 = doc/manpages/swift.1 +[extras] +keystone = + python-keystoneclient>=0.7.0 + [entry_points] console_scripts = swift = swiftclient.shell:main diff --git a/swiftclient/__init__.py b/swiftclient/__init__.py index b412f13..dc192af 100644 --- a/swiftclient/__init__.py +++ b/swiftclient/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2012 Rackspace -# flake8: noqa +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -17,7 +17,7 @@ """ OpenStack Swift Python client binding. """ -from .client import * +from .client import * # noqa # At setup.py time, we haven't installed anything yet, so there # is nothing that is able to set this version property. Squelching diff --git a/swiftclient/client.py b/swiftclient/client.py index 8844a53..744a876 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -72,6 +72,69 @@ if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): logger = logging.getLogger("swiftclient") logger.addHandler(NullHandler()) +#: Default behaviour is to redact header values known to contain secrets, +#: such as ``X-Auth-Key`` and ``X-Auth-Token``. Up to the first 16 chars +#: may be revealed. +#: +#: To disable, set the value of ``redact_sensitive_headers`` to ``False``. +#: +#: When header redaction is enabled, ``reveal_sensitive_prefix`` configures the +#: maximum length of any sensitive header data sent to the logs. If the header +#: is less than twice this length, only ``int(len(value)/2)`` chars will be +#: logged; if it is less than 15 chars long, even less will be logged. +logger_settings = { + 'redact_sensitive_headers': True, + 'reveal_sensitive_prefix': 16 +} +#: A list of sensitive headers to redact in logs. Note that when extending this +#: list, the header names must be added in all lower case. +LOGGER_SENSITIVE_HEADERS = [ + 'x-auth-token', 'x-auth-key', 'x-service-token', 'x-storage-token', + 'x-account-meta-temp-url-key', 'x-account-meta-temp-url-key-2', + 'x-container-meta-temp-url-key', 'x-container-meta-temp-url-key-2', + 'set-cookie' +] + + +def safe_value(name, value): + """ + Only show up to logger_settings['reveal_sensitive_prefix'] characters + from a sensitive header. + + :param name: Header name + :param value: Header value + :return: Safe header value + """ + if name.lower() in LOGGER_SENSITIVE_HEADERS: + prefix_length = logger_settings.get('reveal_sensitive_prefix', 16) + prefix_length = int( + min(prefix_length, (len(value) ** 2) / 32, len(value) / 2) + ) + redacted_value = value[0:prefix_length] + return redacted_value + '...' + return value + + +def scrub_headers(headers): + """ + Redact header values that can contain sensitive information that + should not be logged. + + :param headers: Either a dict or an iterable of two-element tuples + :return: Safe dictionary of headers with sensitive information removed + """ + if isinstance(headers, dict): + headers = headers.items() + headers = [ + (parse_header_string(key), parse_header_string(val)) + for (key, val) in headers + ] + if not logger_settings.get('redact_sensitive_headers', True): + return dict(headers) + if logger_settings.get('reveal_sensitive_prefix', 16) < 0: + logger_settings['reveal_sensitive_prefix'] = 16 + return {key: safe_value(key, val) for (key, val) in headers} + def http_log(args, kwargs, resp, body): if not logger.isEnabledFor(logging.INFO): @@ -87,8 +150,9 @@ def http_log(args, kwargs, resp, body): else: string_parts.append(' %s' % element) if 'headers' in kwargs: - for element in kwargs['headers']: - header = ' -H "%s: %s"' % (element, kwargs['headers'][element]) + headers = scrub_headers(kwargs['headers']) + for element in headers: + header = ' -H "%s: %s"' % (element, headers[element]) string_parts.append(header) # log response as debug if good, or info if error @@ -99,12 +163,14 @@ def http_log(args, kwargs, resp, body): log_method("REQ: %s", "".join(string_parts)) log_method("RESP STATUS: %s %s", resp.status, resp.reason) - log_method("RESP HEADERS: %s", resp.getheaders()) + log_method("RESP HEADERS: %s", scrub_headers(resp.getheaders())) if body: log_method("RESP BODY: %s", body) def parse_header_string(data): + if not isinstance(data, (six.text_type, six.binary_type)): + data = str(data) if six.PY2: if isinstance(data, six.text_type): # Under Python2 requests only returns binary_type, but if we get @@ -202,15 +268,13 @@ class _RetryBody(_ObjectBody): (from offset) if the connection is dropped after partially downloading the object. """ - def __init__(self, resp, expected_length, etag, connection, container, obj, + def __init__(self, resp, connection, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): """ Wrap the underlying response :param resp: the response to wrap - :param expected_length: the object size in bytes - :param etag: the object's etag :param connection: Connection class instance :param container: the name of the container the object is in :param obj: the name of object we are downloading @@ -222,8 +286,7 @@ class _RetryBody(_ObjectBody): include in the request """ super(_RetryBody, self).__init__(resp, resp_chunk_size) - self.expected_length = expected_length - self.expected_etag = etag + self.expected_length = int(self.resp.getheader('Content-Length')) self.conn = connection self.container = container self.obj = obj @@ -244,7 +307,7 @@ class _RetryBody(_ObjectBody): if (not buf and self.bytes_read < self.expected_length and self.conn.attempts <= self.conn.retries): self.headers['Range'] = 'bytes=%d-' % self.bytes_read - self.headers['If-Match'] = self.expected_etag + self.headers['If-Match'] = self.resp.getheader('ETag') hdrs, body = self.conn._retry(None, get_object, self.container, self.obj, resp_chunk_size=self.chunk_size, @@ -252,14 +315,32 @@ class _RetryBody(_ObjectBody): response_dict=self.response_dict, headers=self.headers, attempts=self.conn.attempts) - self.resp = body + expected_range = 'bytes %d-%d/%d' % ( + self.bytes_read, + self.expected_length - 1, + self.expected_length) + if 'content-range' not in hdrs: + # Server didn't respond with partial content; manually seek + logger.warning('Received 200 while retrying %s/%s; seeking...', + self.container, self.obj) + to_read = self.bytes_read + while to_read > 0: + buf = body.resp.read(min(to_read, self.chunk_size)) + to_read -= len(buf) + elif hdrs['content-range'] != expected_range: + msg = ('Expected range "%s" while retrying %s/%s ' + 'but got "%s"' % (expected_range, self.container, + self.obj, hdrs['content-range'])) + raise ClientException(msg) + self.resp = body.resp buf = self.read(length) return buf class HTTPConnection(object): def __init__(self, url, proxy=None, cacert=None, insecure=False, - ssl_compression=False, default_user_agent=None, timeout=None): + cert=None, cert_key=None, ssl_compression=False, + default_user_agent=None, timeout=None): """ Make an HTTPConnection or HTTPSConnection @@ -270,6 +351,9 @@ class HTTPConnection(object): certificate. :param insecure: Allow to access servers without checking SSL certs. The server's certificate will not be verified. + :param cert: Client certificate file to connect on SSL server + requiring SSL client certificate. + :param cert_key: Client certificate private key file. :param ssl_compression: SSL compression should be disabled by default and this setting is not usable as of now. The parameter is kept for backward compatibility. @@ -299,6 +383,14 @@ class HTTPConnection(object): # verify requests parameter is used to pass the CA_BUNDLE file # see: http://docs.python-requests.org/en/latest/user/advanced/ self.requests_args['verify'] = cacert + if cert: + # NOTE(cbrandily): cert requests parameter is used to pass client + # cert path or a tuple with client certificate/key paths. + if cert_key: + self.requests_args['cert'] = cert, cert_key + else: + self.requests_args['cert'] = cert + if proxy: proxy_parsed = urlparse(proxy) if not proxy_parsed.scheme: @@ -385,24 +477,25 @@ def http_connection(*arg, **kwarg): def get_auth_1_0(url, user, key, snet, **kwargs): cacert = kwargs.get('cacert', None) insecure = kwargs.get('insecure', False) + cert = kwargs.get('cert') + cert_key = kwargs.get('cert_key') timeout = kwargs.get('timeout', None) parsed, conn = http_connection(url, cacert=cacert, insecure=insecure, + cert=cert, cert_key=cert_key, timeout=timeout) method = 'GET' - conn.request(method, parsed.path, '', - {'X-Auth-User': user, 'X-Auth-Key': key}) + headers = {'X-Auth-User': user, 'X-Auth-Key': key} + conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {}, resp, body) + http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') # There is a side-effect on current Rackspace 1.0 server where a # bad URL would get you that document page and a 200. We error out # if we don't have a x-storage-url header and if we get a body. if resp.status < 200 or resp.status >= 300 or (body and not url): - raise ClientException('Auth GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason) + raise ClientException.from_response(resp, 'Auth GET failed', body) if snet: parsed = list(urlparse(url)) # Second item in the list is the netloc @@ -457,6 +550,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): _ksclient = ksclient.Client( username=user, password=key, + token=os_options.get('auth_token'), tenant_name=os_options.get('tenant_name'), tenant_id=os_options.get('tenant_id'), user_id=os_options.get('user_id'), @@ -468,6 +562,8 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): project_domain_id=os_options.get('project_domain_id'), debug=debug, cacert=kwargs.get('cacert'), + cert=kwargs.get('cert'), + key=kwargs.get('cert_key'), auth_url=auth_url, insecure=insecure, timeout=timeout) except exceptions.Unauthorized: msg = 'Unauthorized. Check username, password and tenant name/id.' @@ -518,6 +614,8 @@ def get_auth(auth_url, user, key, **kwargs): cacert = kwargs.get('cacert', None) insecure = kwargs.get('insecure', False) + cert = kwargs.get('cert') + cert_key = kwargs.get('cert_key') timeout = kwargs.get('timeout', None) if auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, @@ -526,6 +624,8 @@ def get_auth(auth_url, user, key, **kwargs): kwargs.get('snet'), cacert=cacert, insecure=insecure, + cert=cert, + cert_key=cert_key, timeout=timeout) elif auth_version in AUTH_VERSIONS_V2 + AUTH_VERSIONS_V3: # We are handling a special use case here where the user argument @@ -549,6 +649,8 @@ def get_auth(auth_url, user, key, **kwargs): key, os_options, cacert=cacert, insecure=insecure, + cert=cert, + cert_key=cert_key, timeout=timeout, auth_version=auth_version) else: @@ -597,8 +699,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, :param limit: limit query :param prefix: prefix query :param end_marker: end_marker query - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings :param service_token: service auth token @@ -641,11 +743,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, resp_headers = resp_header_dict(resp) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_query=qs, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account GET failed', body) if resp.status == 204: return resp_headers, [] return resp_headers, parse_api_response(resp_headers, body) @@ -657,8 +755,8 @@ def head_account(url, token, http_conn=None, service_token=None): :param url: storage URL :param token: auth token - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) @@ -677,10 +775,7 @@ def head_account(url, token, http_conn=None, service_token=None): body = resp.read() http_log((url, method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -693,8 +788,8 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, :param url: storage URL :param token: auth token :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token @@ -722,13 +817,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account POST failed', - http_scheme=parsed.scheme, - http_host=conn.host, - http_path=parsed.path, - http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account POST failed', body) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value @@ -751,8 +840,8 @@ def get_container(url, token, container, marker=None, limit=None, :param delimiter: string to delimit the queries on :param end_marker: marker query :param path: path query (equivalent: "delimiter=/" and "prefix=path/") - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings :param service_token: service auth token @@ -771,7 +860,7 @@ def get_container(url, token, container, marker=None, limit=None, if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, - service_token, headers=headers) + service_token=service_token, headers=headers) listing = rv[1] while listing: if not delimiter: @@ -780,7 +869,7 @@ def get_container(url, token, container, marker=None, limit=None, marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, - http_conn, service_token, + http_conn, service_token=service_token, headers=headers)[1] if listing: rv[1].extend(listing) @@ -813,11 +902,7 @@ def get_container(url, token, container, marker=None, limit=None, {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container GET failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=cont_path, http_query=qs, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Container GET failed', body) resp_headers = resp_header_dict(resp) if resp.status == 204: return resp_headers, [] @@ -832,8 +917,8 @@ def head_container(url, token, container, http_conn=None, headers=None, :param url: storage URL :param token: auth token :param container: container name to get stats for - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param headers: additional headers to include in the request :param service_token: service auth token :returns: a dict containing the response's headers (all header names will @@ -858,11 +943,8 @@ def head_container(url, token, container, http_conn=None, headers=None, {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container HEAD failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -876,8 +958,8 @@ def put_container(url, token, container, headers=None, http_conn=None, :param token: auth token :param container: container name to create :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token @@ -905,11 +987,7 @@ def put_container(url, token, container, headers=None, http_conn=None, http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container PUT failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Container PUT failed', body) def post_container(url, token, container, headers, http_conn=None, @@ -921,8 +999,8 @@ def post_container(url, token, container, headers, http_conn=None, :param token: auth token :param container: container name to update :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token @@ -948,11 +1026,8 @@ def post_container(url, token, container, headers, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container POST failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container POST failed', body) def delete_container(url, token, container, http_conn=None, @@ -963,8 +1038,8 @@ def delete_container(url, token, container, http_conn=None, :param url: storage URL :param token: auth token :param container: container name to delete - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token @@ -988,11 +1063,8 @@ def delete_container(url, token, container, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container DELETE failed', body) def get_object(url, token, container, name, http_conn=None, @@ -1005,8 +1077,8 @@ def get_object(url, token, container, name, http_conn=None, :param token: auth token :param container: container name that the object is in :param name: object name to get - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If you specify a resp_chunk_size you must fully read the object's contents before making another @@ -1045,11 +1117,7 @@ def get_object(url, token, container, name, http_conn=None, body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) - raise ClientException('Object GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object GET failed', body) if resp_chunk_size: object_body = _ObjectBody(resp, resp_chunk_size) else: @@ -1069,8 +1137,8 @@ def head_object(url, token, container, name, http_conn=None, :param token: auth token :param container: container name that the object is in :param name: object name to get info for - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param service_token: service auth token :param headers: additional headers to include in the request :returns: a dict containing the response's headers (all header names will @@ -1096,10 +1164,7 @@ def head_object(url, token, container, name, http_conn=None, http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -1134,8 +1199,8 @@ def put_object(url, token=None, container=None, name=None, contents=None, value is found in the headers param, an empty string value will be sent :param headers: additional headers to include in the request, if any - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param proxy: proxy to connect through, if any; None by default; str of the format 'http://127.0.0.1:8888' to set one :param query_string: if set will be appended with '?' to generated path @@ -1211,10 +1276,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object PUT failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object PUT failed', body) etag = resp.getheader('etag', '').strip('"') return etag @@ -1230,8 +1292,8 @@ def post_object(url, token, container, name, headers, http_conn=None, :param container: container name that the object is in :param name: name of the object to update :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token @@ -1254,10 +1316,7 @@ def post_object(url, token, container, name, headers, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object POST failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object POST failed', body) def delete_object(url, token=None, container=None, name=None, http_conn=None, @@ -1272,8 +1331,8 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, container name is expected to be part of the url :param name: object name to delete; if None, the object name is expected to be part of the url - :param http_conn: HTTP connection object (If None, it will create the - conn object) + :param http_conn: a tuple of (parsed url, HTTPConnection object), + (If None, it will create the conn object) :param headers: additional headers to include in the request :param proxy: proxy to connect through, if any; None by default; str of the format 'http://127.0.0.1:8888' to set one @@ -1311,18 +1370,14 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object DELETE failed', body) def get_capabilities(http_conn): """ Get cluster capability infos. - :param http_conn: HTTP connection + :param http_conn: a tuple of (parsed url, HTTPConnection object) :returns: a dict containing the cluster capabilities :raises ClientException: HTTP Capabilities GET failed """ @@ -1332,11 +1387,8 @@ def get_capabilities(http_conn): body = resp.read() http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Capabilities GET failed', - http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Capabilities GET failed', body) resp_headers = resp_header_dict(resp) return parse_api_response(resp_headers, body) @@ -1360,8 +1412,9 @@ class Connection(object): preauthurl=None, preauthtoken=None, snet=False, starting_backoff=1, max_backoff=64, tenant_name=None, os_options=None, auth_version="1", cacert=None, - insecure=False, ssl_compression=True, - retry_on_ratelimit=False, timeout=None): + insecure=False, cert=None, cert_key=None, + ssl_compression=True, retry_on_ratelimit=False, + timeout=None): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1383,6 +1436,9 @@ class Connection(object): service_username, service_project_name, service_key :param insecure: Allow to access servers without checking SSL certs. The server's certificate will not be verified. + :param cert: Client certificate file to connect on SSL server + requiring SSL client certificate. + :param cert_key: Client certificate private key file. :param ssl_compression: Whether to enable compression at the SSL layer. If set to 'False' and the pyOpenSSL library is present an attempt to disable SSL compression @@ -1418,6 +1474,8 @@ class Connection(object): self.service_token = None self.cacert = cacert self.insecure = insecure + self.cert = cert + self.cert_key = cert_key self.ssl_compression = ssl_compression self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit @@ -1440,6 +1498,8 @@ class Connection(object): os_options=self.os_options, cacert=self.cacert, insecure=self.insecure, + cert=self.cert, + cert_key=self.cert_key, timeout=self.timeout) return self.url, self.token @@ -1465,6 +1525,8 @@ class Connection(object): return http_connection(url if url else self.url, cacert=self.cacert, insecure=self.insecure, + cert=self.cert, + cert_key=self.cert_key, ssl_compression=self.ssl_compression, timeout=self.timeout) @@ -1604,11 +1666,10 @@ class Connection(object): not headers or 'range' not in (k.lower() for k in headers)) retry_is_possible = ( is_not_range_request and resp_chunk_size and - self.attempts <= self.retries) + self.attempts <= self.retries and + rheaders.get('transfer-encoding') is None) if retry_is_possible: - body = _RetryBody(body.resp, int(rheaders['content-length']), - rheaders['etag'], - self, container, obj, + body = _RetryBody(body.resp, self, container, obj, resp_chunk_size=resp_chunk_size, query_string=query_string, response_dict=response_dict, diff --git a/swiftclient/exceptions.py b/swiftclient/exceptions.py index 370a8d0..da70379 100644 --- a/swiftclient/exceptions.py +++ b/swiftclient/exceptions.py @@ -13,12 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from six.moves import urllib + class ClientException(Exception): def __init__(self, msg, http_scheme='', http_host='', http_port='', http_path='', http_query='', http_status=None, http_reason='', - http_device='', http_response_content=''): + http_device='', http_response_content='', + http_response_headers=None): super(ClientException, self).__init__(msg) self.msg = msg self.http_scheme = http_scheme @@ -30,6 +33,16 @@ class ClientException(Exception): self.http_reason = http_reason self.http_device = http_device self.http_response_content = http_response_content + self.http_response_headers = http_response_headers + + @classmethod + def from_response(cls, resp, msg=None, body=None): + msg = msg or '%s %s' % (resp.status_code, resp.reason) + body = body or resp.content + parsed_url = urllib.parse.urlparse(resp.request.url) + return cls(msg, parsed_url.scheme, parsed_url.hostname, + parsed_url.port, parsed_url.path, parsed_url.query, + resp.status_code, resp.reason, '', body, resp.headers) def __str__(self): a = self.msg diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index 2778fac..5e03ed7 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -126,7 +126,6 @@ class MultiThreadingManager(object): def __init__(self, create_connection, segment_threads=10, object_dd_threads=10, object_uu_threads=10, container_threads=10): - """ :param segment_threads: The number of threads allocated to segment uploads diff --git a/swiftclient/service.py b/swiftclient/service.py index 20c023b..12d3f21 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -25,6 +25,7 @@ from os import environ, makedirs, stat, utime from os.path import ( basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep ) +from posixpath import join as urljoin from random import shuffle from time import time from threading import Thread @@ -85,12 +86,25 @@ class SwiftError(Exception): def process_options(options): - if (not (options.get('auth') and options.get('user') - and options.get('key')) - and options.get('auth_version') != '3'): - # Use keystone 2.0 auth if any of the old-style args are missing + # tolerate sloppy auth_version + if options.get('auth_version') == '3.0': + options['auth_version'] = '3' + elif options.get('auth_version') == '2': options['auth_version'] = '2.0' + if options.get('auth_version') not in ('2.0', '3') and not all( + options.get(key) for key in ('auth', 'user', 'key')): + # Use keystone auth if any of the new-style args are present + if any(options.get(k) for k in ( + 'os_user_domain_id', + 'os_user_domain_name', + 'os_project_domain_id', + 'os_project_domain_name')): + # Use v3 if there's any reference to domains + options['auth_version'] = '3' + else: + options['auth_version'] = '2.0' + # Use new-style args if old ones not present if not options['auth'] and options['os_auth_url']: options['auth'] = options['os_auth_url'] @@ -147,6 +161,8 @@ def _build_default_global_options(): "os_service_type": environ.get('OS_SERVICE_TYPE'), "os_endpoint_type": environ.get('OS_ENDPOINT_TYPE'), "os_cacert": environ.get('OS_CACERT'), + "os_cert": environ.get('OS_CERT'), + "os_key": environ.get('OS_KEY'), "insecure": config_true_value(environ.get('SWIFTCLIENT_INSECURE')), "ssl_compression": False, 'segment_threads': 10, @@ -188,6 +204,10 @@ _default_local_options = { } POLICY = 'X-Storage-Policy' +KNOWN_DIR_MARKERS = ( + 'application/directory', # Preferred + 'text/directory', # Historically relevant +) def get_from_queue(q, timeout=864000): @@ -235,6 +255,8 @@ def get_conn(options): snet=options['snet'], cacert=options['os_cacert'], insecure=options['insecure'], + cert=options['os_cert'], + cert_key=options['os_key'], ssl_compression=options['ssl_compression']) @@ -288,6 +310,7 @@ class SwiftUploadObject(object): if not self.object_name: raise SwiftError('Object names must not be empty strings') + self.object_name = self.object_name.lstrip('/') self.options = options self.source = source @@ -567,7 +590,7 @@ class SwiftService(object): { 'meta': [], - 'headers': [], + 'header': [], 'read_acl': None, # For containers only 'write_acl': None, # For containers only 'sync_to': None, # For containers only @@ -698,10 +721,10 @@ class SwiftService(object): if 'meta' in obj_options: headers.update( split_headers( - obj_options['meta'], 'X-Object-Meta' + obj_options['meta'], 'X-Object-Meta-' ) ) - if 'headers' in obj_options: + if 'header' in obj_options: headers.update( split_headers(obj_options['header'], '') ) @@ -881,7 +904,7 @@ class SwiftService(object): @staticmethod def _list_container_job(conn, container, options, result_queue): - marker = '' + marker = options.get('marker', '') error = None try: while True: @@ -1002,7 +1025,7 @@ class SwiftService(object): raise raise SwiftError('Account not found', exc=err) - elif not objects: + elif objects is None: if '/' in container: raise SwiftError('\'/\' in container name', container=container) @@ -1128,9 +1151,8 @@ class SwiftService(object): fp = None try: - content_type = headers.get('content-type') - if (content_type and - content_type.split(';', 1)[0] == 'text/directory'): + content_type = headers.get('content-type', '').split(';', 1)[0] + if content_type in KNOWN_DIR_MARKERS: make_dir = not no_file and out_file != "-" if make_dir and not isdir(path): mkdirs(path) @@ -1288,7 +1310,8 @@ class SwiftService(object): """ Upload a list of objects to a given container. - :param container: The container to put the uploads into. + :param container: The container (or pseudo-folder path) to put the + uploads into. :param objects: A list of file/directory names (strings) or SwiftUploadObject instances containing a source for the created object, an object name, and an options dict @@ -1346,10 +1369,9 @@ class SwiftService(object): raise SwiftError('Segment size should be an integer value') # Incase we have a psudeo-folder path for <container> arg, derive - # the container name from the top path to ensure new folder creation - # and prevent spawning zero-byte objects shadowing pseudo-folders - # by name. - container_name = container.split('/', 1)[0] + # the container name from the top path and prepend the rest to + # the object name. (same as passing --object-name). + container, _sep, pseudo_folder = container.partition('/') # Try to create the container, just in case it doesn't exist. If this # fails, it might just be because the user doesn't have container PUT @@ -1362,10 +1384,7 @@ class SwiftService(object): _header[POLICY] create_containers = [ self.thread_manager.container_pool.submit( - self._create_container_job, - container_name, - headers=policy_header - ) + self._create_container_job, container, headers=policy_header) ] # wait for first container job to complete before possibly attempting @@ -1409,7 +1428,7 @@ class SwiftService(object): rq = Queue() file_jobs = {} - upload_objects = self._make_upload_objects(objects) + upload_objects = self._make_upload_objects(objects, pseudo_folder) for upload_object in upload_objects: s = upload_object.source o = upload_object.object_name @@ -1500,14 +1519,16 @@ class SwiftService(object): res = get_from_queue(rq) @staticmethod - def _make_upload_objects(objects): + def _make_upload_objects(objects, pseudo_folder=''): upload_objects = [] for o in objects: if isinstance(o, string_types): - obj = SwiftUploadObject(o) + obj = SwiftUploadObject(o, urljoin(pseudo_folder, + o.lstrip('/'))) upload_objects.append(obj) elif isinstance(o, SwiftUploadObject): + o.object_name = urljoin(pseudo_folder, o.object_name) upload_objects.append(o) else: raise SwiftError( @@ -1589,12 +1610,12 @@ class SwiftService(object): if options['changed']: try: headers = conn.head_object(container, obj) - ct = headers.get('content-type') + ct = headers.get('content-type', '').split(';', 1)[0] cl = int(headers.get('content-length')) et = headers.get('etag') mt = headers.get('x-object-meta-mtime') - if (ct.split(';', 1)[0] == 'text/directory' and + if (ct in KNOWN_DIR_MARKERS and cl == 0 and et == EMPTY_ETAG and mt == put_headers['x-object-meta-mtime']): @@ -1613,7 +1634,7 @@ class SwiftService(object): return res try: conn.put_object(container, obj, '', content_length=0, - content_type='text/directory', + content_type=KNOWN_DIR_MARKERS[0], headers=put_headers, response_dict=results_dict) res.update({ @@ -1656,10 +1677,13 @@ class SwiftService(object): fp.seek(segment_start) contents = LengthWrapper(fp, segment_size, md5=options['checksum']) - etag = conn.put_object(segment_container, - segment_name, contents, - content_length=segment_size, - response_dict=results_dict) + etag = conn.put_object( + segment_container, + segment_name, + contents, + content_length=segment_size, + content_type='application/swiftclient-segment', + response_dict=results_dict) if options['checksum'] and etag and etag != contents.get_md5sum(): raise SwiftError('Segment {0}: upload verification failed: ' diff --git a/swiftclient/shell.py b/swiftclient/shell.py index e427a89..5eafe0b 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -16,11 +16,11 @@ from __future__ import print_function, unicode_literals +import argparse import logging import signal import socket -from optparse import OptionParser, OptionGroup, SUPPRESS_HELP from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join from six import text_type, PY2 @@ -33,8 +33,10 @@ from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version +from swiftclient.client import logger_settings as client_logger_settings, \ + parse_header_string from swiftclient.service import SwiftService, SwiftError, \ - SwiftUploadObject, get_conn + SwiftUploadObject, get_conn, process_options from swiftclient.command_helpers import print_account_stats, \ print_container_stats, print_object_stats @@ -79,35 +81,50 @@ Optional arguments: def st_delete(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') - parser.add_option( + parser.add_argument( '-p', '--prefix', dest='prefix', help='Only delete items beginning with the <prefix>.') - parser.add_option( - '', '--leave-segments', action='store_true', + parser.add_argument( + '--leave-segments', action='store_true', dest='leave_segments', default=False, help='Do not delete segments of manifest objects.') - parser.add_option( - '', '--object-threads', type=int, + parser.add_argument( + '--object-threads', type=int, default=10, help='Number of threads to use for deleting objects. ' - 'Default is 10.') - parser.add_option( - '', '--container-threads', type=int, + 'Its value must be a positive integer. Default is 10.') + parser.add_argument( + '--container-threads', type=int, default=10, help='Number of threads to use for deleting containers. ' - 'Default is 10.') + 'Its value must be a positive integer. Default is 10.') (options, args) = parse_args(parser, args) args = args[1:] - if (not args and not options.yes_all) or (args and options.yes_all): + if (not args and not options['yes_all']) or (args and options['yes_all']): output_manager.error('Usage: %s delete %s\n%s', BASENAME, st_delete_options, st_delete_help) return - _opts = vars(options) - _opts['object_dd_threads'] = options.object_threads - with SwiftService(options=_opts) as swift: + if options['object_threads'] <= 0: + output_manager.error( + 'ERROR: option --object-threads should be a positive integer.' + '\n\nUsage: %s delete %s\n%s', + BASENAME, st_delete_options, + st_delete_help) + return + + if options['container_threads'] <= 0: + output_manager.error( + 'ERROR: option --container-threads should be a positive integer.' + '\n\nUsage: %s delete %s\n%s', + BASENAME, st_delete_options, + st_delete_help) + return + + options['object_dd_threads'] = options['object_threads'] + with SwiftService(options=options) as swift: try: if not args: del_iter = swift.delete() @@ -151,7 +168,7 @@ def st_delete(parser, args, output_manager): pass for o in objs: - if options.yes_all: + if options['yes_all']: p = '{0}/{1}'.format(c, o) else: p = o @@ -162,12 +179,12 @@ def st_delete(parser, args, output_manager): .format(c, o, r['error'])) else: if r['success']: - if options.verbose: + if options['verbose']: a = (' [after {0} attempts]'.format(a) if a > 1 else '') if r['action'] == 'delete_object': - if options.yes_all: + if options['yes_all']: p = '{0}/{1}'.format(c, o) else: p = o @@ -185,7 +202,7 @@ def st_delete(parser, args, output_manager): output_manager.error(err.value) -st_download_options = '''[--all] [--marker] [--prefix <prefix>] +st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>] [--output <out_file>] [--output-dir <out_directory>] [--object-threads <threads>] [--container-threads <threads>] [--no-download] @@ -207,7 +224,7 @@ Positional arguments: Optional arguments: -a, --all Indicates that you really want to download everything in the account. - -m, --marker Marker to use when starting a container or account + -m, --marker <marker> Marker to use when starting a container or account download. -p, --prefix <prefix> Only download items beginning with <prefix> -r, --remove-prefix An optional flag for --prefix <prefix>, use this @@ -236,7 +253,7 @@ Optional arguments: sides. --no-shuffle By default, when downloading a complete account or container, download order is randomised in order to - to reduce the load on individual drives when multiple + reduce the load on individual drives when multiple clients are executed simultaneously to download the same set of objects (e.g. a nightly automated download script to multiple servers). Enable this option to @@ -246,52 +263,52 @@ Optional arguments: def st_download(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Indicates that you really want to download ' 'everything in the account.') - parser.add_option( + parser.add_argument( '-m', '--marker', dest='marker', default='', help='Marker to use when starting a container or ' 'account download.') - parser.add_option( + parser.add_argument( '-p', '--prefix', dest='prefix', help='Only download items beginning with the <prefix>.') - parser.add_option( + parser.add_argument( '-o', '--output', dest='out_file', help='For a single ' 'download, stream the output to <out_file>. ' 'Specifying "-" as <out_file> will redirect to stdout.') - parser.add_option( + parser.add_argument( '-D', '--output-dir', dest='out_directory', help='An optional directory to which to store objects. ' 'By default, all objects are recreated in the current directory.') - parser.add_option( + parser.add_argument( '-r', '--remove-prefix', action='store_true', dest='remove_prefix', default=False, help='An optional flag for --prefix <prefix>, ' 'use this option to download items without <prefix>.') - parser.add_option( - '', '--object-threads', type=int, + parser.add_argument( + '--object-threads', type=int, default=10, help='Number of threads to use for downloading objects. ' - 'Default is 10.') - parser.add_option( - '', '--container-threads', type=int, default=10, + 'Its value must be a positive integer. Default is 10.') + parser.add_argument( + '--container-threads', type=int, default=10, help='Number of threads to use for downloading containers. ' - 'Default is 10.') - parser.add_option( - '', '--no-download', action='store_true', + 'Its value must be a positive integer. Default is 10.') + parser.add_argument( + '--no-download', action='store_true', default=False, help="Perform download(s), but don't actually write anything to disk.") - parser.add_option( + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Adds a customized request header to the query, like "Range" or ' '"If-Match". This option may be repeated. ' 'Example: --header "content-type:text/plain"') - parser.add_option( + parser.add_argument( '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip downloading files that are identical on ' 'both sides.') - parser.add_option( + parser.add_argument( '--no-shuffle', action='store_false', dest='shuffle', default=True, help='By default, download order is randomised in order ' 'to reduce the load on individual drives when multiple clients are ' @@ -301,26 +318,39 @@ def st_download(parser, args, output_manager): 'are listed in the object store.') (options, args) = parse_args(parser, args) args = args[1:] - if options.out_file == '-': - options.verbose = 0 + if options['out_file'] == '-': + options['verbose'] = 0 - if options.out_file and len(args) != 2: + if options['out_file'] and len(args) != 2: exit('-o option only allowed for single file downloads') - if not options.prefix: - options.remove_prefix = False + if not options['prefix']: + options['remove_prefix'] = False - if options.out_directory and len(args) == 2: + if options['out_directory'] and len(args) == 2: exit('Please use -o option for single file downloads and renames') - if (not args and not options.yes_all) or (args and options.yes_all): + if (not args and not options['yes_all']) or (args and options['yes_all']): output_manager.error('Usage: %s download %s\n%s', BASENAME, st_download_options, st_download_help) return - _opts = vars(options) - _opts['object_dd_threads'] = options.object_threads - with SwiftService(options=_opts) as swift: + if options['object_threads'] <= 0: + output_manager.error( + 'ERROR: option --object-threads should be a positive integer.\n\n' + 'Usage: %s download %s\n%s', BASENAME, + st_download_options, st_download_help) + return + + if options['container_threads'] <= 0: + output_manager.error( + 'ERROR: option --container-threads should be a positive integer.' + '\n\nUsage: %s download %s\n%s', BASENAME, + st_download_options, st_download_help) + return + + options['object_dd_threads'] = options['object_threads'] + with SwiftService(options=options) as swift: try: if not args: down_iter = swift.download() @@ -340,13 +370,13 @@ def st_download(parser, args, output_manager): down_iter = swift.download(container, objects) for down in down_iter: - if options.out_file == '-' and 'contents' in down: + if options['out_file'] == '-' and 'contents' in down: contents = down['contents'] for chunk in contents: output_manager.print_raw(chunk) else: if down['success']: - if options.verbose: + if options['verbose']: start_time = down['start_time'] headers_receipt = \ down['headers_receipt'] - start_time @@ -391,7 +421,7 @@ def st_download(parser, args, output_manager): obj = down['object'] if isinstance(error, ClientException): if error.http_status == 304 and \ - options.skip_identical: + options['skip_identical']: output_manager.print_msg( "Skipped identical file '%s'", path) continue @@ -435,17 +465,17 @@ Optional arguments: def st_list(parser, args, output_manager): - def _print_stats(options, stats): + def _print_stats(options, stats, human): total_count = total_bytes = 0 container = stats.get("container", None) for item in stats["listing"]: item_name = item.get('name') - if not options.long and not options.human: + if not options['long'] and not human: output_manager.print_msg(item.get('name', item.get('subdir'))) else: if not container: # listing containers item_bytes = item.get('bytes') - byte_str = prt_bytes(item_bytes, options.human) + byte_str = prt_bytes(item_bytes, human) count = item.get('count') total_count += count try: @@ -454,7 +484,7 @@ def st_list(parser, args, output_manager): datestamp = strftime('%Y-%m-%d %H:%M:%S', utc) except TypeError: datestamp = '????-??-?? ??:??:??' - if not options.totals: + if not options['totals']: output_manager.print_msg( "%5s %s %s %s", count, byte_str, datestamp, item_name) @@ -463,65 +493,64 @@ def st_list(parser, args, output_manager): content_type = item.get('content_type') if subdir is None: item_bytes = item.get('bytes') - byte_str = prt_bytes(item_bytes, options.human) + byte_str = prt_bytes(item_bytes, human) date, xtime = item.get('last_modified').split('T') xtime = xtime.split('.')[0] else: item_bytes = 0 - byte_str = prt_bytes(item_bytes, options.human) + byte_str = prt_bytes(item_bytes, human) date = xtime = '' item_name = subdir - if not options.totals: + if not options['totals']: output_manager.print_msg( "%s %10s %8s %24s %s", byte_str, date, xtime, content_type, item_name) total_bytes += item_bytes # report totals - if options.long or options.human: + if options['long'] or human: if not container: output_manager.print_msg( "%5s %s", prt_bytes(total_count, True), - prt_bytes(total_bytes, options.human)) + prt_bytes(total_bytes, human)) else: output_manager.print_msg( - prt_bytes(total_bytes, options.human)) + prt_bytes(total_bytes, human)) - parser.add_option( + parser.add_argument( '-l', '--long', dest='long', action='store_true', default=False, help='Long listing format, similar to ls -l.') - parser.add_option( + parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format, ' "similar to ls -lh.") - parser.add_option( + parser.add_argument( '-t', '--totals', dest='totals', help='used with -l or --lh, only report totals.', action='store_true', default=False) - parser.add_option( + parser.add_argument( '-p', '--prefix', dest='prefix', help='Only list items beginning with the prefix.') - parser.add_option( + parser.add_argument( '-d', '--delimiter', dest='delimiter', help='Roll up items with the given delimiter. For containers ' 'only. See OpenStack Swift API documentation for ' 'what this means.') - (options, args) = parse_args(parser, args) + options, args = parse_args(parser, args) args = args[1:] - if options.delimiter and not args: + if options['delimiter'] and not args: exit('-d option only allowed for container listings') - _opts = vars(options).copy() - if _opts['human']: - _opts.pop('human') - _opts['long'] = True + human = options.pop('human') + if human: + options['long'] = True - if options.totals and not options.long and not options.human: + if options['totals'] and not options['long']: output_manager.error( "Listing totals only works with -l or --lh.") return - with SwiftService(options=_opts) as swift: + with SwiftService(options=options) as swift: try: if not args: stats_parts_gen = swift.list() @@ -538,7 +567,7 @@ def st_list(parser, args, output_manager): for stats in stats_parts_gen: if stats["success"]: - _print_stats(options, stats) + _print_stats(options, stats, human) else: raise stats["error"] @@ -564,15 +593,13 @@ Optional arguments: def st_stat(parser, args, output_manager): - parser.add_option( + parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') - (options, args) = parse_args(parser, args) + options, args = parse_args(parser, args) args = args[1:] - _opts = vars(options) - - with SwiftService(options=_opts) as swift: + with SwiftService(options=options) as swift: try: if not args: stat_result = swift.stat() @@ -655,25 +682,25 @@ Optional arguments: def st_post(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-r', '--read-acl', dest='read_acl', help='Read ACL for containers. ' 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' '.r:www.example.com, account1, account2:user2') - parser.add_option( + parser.add_argument( '-w', '--write-acl', dest='write_acl', help='Write ACL for ' 'containers. Quick summary of ACL syntax: account1, ' 'account2:user2') - parser.add_option( + parser.add_argument( '-t', '--sync-to', dest='sync_to', help='Sets the ' 'Sync To for containers, for multi-cluster replication.') - parser.add_option( + parser.add_argument( '-k', '--sync-key', dest='sync_key', help='Sets the ' 'Sync Key for containers, for multi-cluster replication.') - parser.add_option( + parser.add_argument( '-m', '--meta', action='append', dest='meta', default=[], help='Sets a meta data item. This option may be repeated. ' 'Example: -m Color:Blue -m Size:Large') - parser.add_option( + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Adds a customized request header. ' 'This option may be repeated. ' @@ -681,13 +708,11 @@ def st_post(parser, args, output_manager): '-H "Content-Length: 4000"') (options, args) = parse_args(parser, args) args = args[1:] - if (options.read_acl or options.write_acl or options.sync_to or - options.sync_key) and not args: + if (options['read_acl'] or options['write_acl'] or options['sync_to'] or + options['sync_key']) and not args: exit('-r, -w, -t, and -k options only allowed for containers') - _opts = vars(options) - - with SwiftService(options=_opts) as swift: + with SwiftService(options=options) as swift: try: if not args: result = swift.post() @@ -774,58 +799,58 @@ Optional arguments: def st_upload(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-c', '--changed', action='store_true', dest='changed', default=False, help='Only upload files that have changed since ' 'the last upload.') - parser.add_option( + parser.add_argument( '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip uploading files that are identical on ' 'both sides.') - parser.add_option( + parser.add_argument( '-S', '--segment-size', dest='segment_size', help='Upload files ' 'in segments no larger than <size> (in Bytes) and then create a ' '"manifest" file that will download all the segments as if it were ' 'the original file. Sizes may also be expressed as bytes with the ' 'B suffix, kilobytes with the K suffix, megabytes with the M suffix ' 'or gigabytes with the G suffix.') - parser.add_option( + parser.add_argument( '-C', '--segment-container', dest='segment_container', help='Upload the segments into the specified container. ' 'If not specified, the segments will be uploaded to a ' '<container>_segments container to not pollute the main ' '<container> listings.') - parser.add_option( - '', '--leave-segments', action='store_true', + parser.add_argument( + '--leave-segments', action='store_true', dest='leave_segments', default=False, help='Indicates that you want ' 'the older segments of manifest objects left alone (in the case of ' 'overwrites).') - parser.add_option( - '', '--object-threads', type=int, default=10, + parser.add_argument( + '--object-threads', type=int, default=10, help='Number of threads to use for uploading full objects. ' - 'Default is 10.') - parser.add_option( - '', '--segment-threads', type=int, default=10, + 'Its value must be a positive integer. Default is 10.') + parser.add_argument( + '--segment-threads', type=int, default=10, help='Number of threads to use for uploading object segments. ' - 'Default is 10.') - parser.add_option( + 'Its value must be a positive integer. Default is 10.') + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' ' This option may be repeated. Example -H "content-type:text/plain" ' '-H "Content-Length: 4000"') - parser.add_option( - '', '--use-slo', action='store_true', default=False, + parser.add_argument( + '--use-slo', action='store_true', default=False, help='When used in conjunction with --segment-size, it will ' 'create a Static Large Object instead of the default ' 'Dynamic Large Object.') - parser.add_option( - '', '--object-name', dest='object_name', + parser.add_argument( + '--object-name', dest='object_name', help='Upload file and name object to <object-name> or upload dir and ' 'use <object-name> as object prefix instead of folder name.') - parser.add_option( - '', '--ignore-checksum', dest='checksum', default=True, + parser.add_argument( + '--ignore-checksum', dest='checksum', default=True, action='store_false', help='Turn off checksum validation for uploads.') - (options, args) = parse_args(parser, args) + options, args = parse_args(parser, args) args = args[1:] if len(args) < 2: output_manager.error( @@ -836,33 +861,46 @@ def st_upload(parser, args, output_manager): container = args[0] files = args[1:] - if options.object_name is not None: + if options['object_name'] is not None: if len(files) > 1: output_manager.error('object-name only be used with 1 file or dir') return else: orig_path = files[0] - if options.segment_size: + if options['segment_size']: try: # If segment size only has digits assume it is bytes - int(options.segment_size) + int(options['segment_size']) except ValueError: try: - size_mod = "BKMG".index(options.segment_size[-1].upper()) - multiplier = int(options.segment_size[:-1]) + size_mod = "BKMG".index(options['segment_size'][-1].upper()) + multiplier = int(options['segment_size'][:-1]) except ValueError: output_manager.error("Invalid segment size") return - options.segment_size = str((1024 ** size_mod) * multiplier) - if int(options.segment_size) <= 0: + options['segment_size'] = str((1024 ** size_mod) * multiplier) + if int(options['segment_size']) <= 0: output_manager.error("segment-size should be positive") return - _opts = vars(options) - _opts['object_uu_threads'] = options.object_threads - with SwiftService(options=_opts) as swift: + if options['object_threads'] <= 0: + output_manager.error( + 'ERROR: option --object-threads should be a positive integer.' + '\n\nUsage: %s upload %s\n%s', BASENAME, st_upload_options, + st_upload_help) + return + + if options['segment_threads'] <= 0: + output_manager.error( + 'ERROR: option --segment-threads should be a positive integer.' + '\n\nUsage: %s upload %s\n%s', BASENAME, st_upload_options, + st_upload_help) + return + + options['object_uu_threads'] = options['object_threads'] + with SwiftService(options=options) as swift: try: objs = [] dir_markers = [] @@ -880,25 +918,25 @@ def st_upload(parser, args, output_manager): # Now that we've collected all the required files and dir markers # build the tuples for the call to upload - if options.object_name is not None: + if options['object_name'] is not None: objs = [ SwiftUploadObject( o, object_name=o.replace( - orig_path, options.object_name, 1 + orig_path, options['object_name'], 1 ) ) for o in objs ] dir_markers = [ SwiftUploadObject( None, object_name=d.replace( - orig_path, options.object_name, 1 + orig_path, options['object_name'], 1 ), options={'dir_marker': True} ) for d in dir_markers ] for r in swift.upload(container, objs + dir_markers): if r['success']: - if options.verbose: + if options['verbose']: if 'attempts' in r and r['attempts'] > 1: if 'object' in r: output_manager.print_msg( @@ -938,13 +976,13 @@ def st_upload(parser, args, output_manager): msg = ': %s' % error output_manager.warning( 'Warning: failed to create container ' - "'%s'%s", container, msg + "'%s'%s", r['container'], msg ) else: output_manager.error("%s" % error) too_large = (isinstance(error, ClientException) and error.http_status == 413) - if too_large and options.verbose > 0: + if too_large and options['verbose'] > 0: output_manager.error( "Consider using the --segment-size option " "to chunk the object") @@ -982,8 +1020,7 @@ def st_capabilities(parser, args, output_manager): st_capabilities_options, st_capabilities_help) return - _opts = vars(options) - with SwiftService(options=_opts) as swift: + with SwiftService(options=options) as swift: try: if len(args) == 2: url = args[1] @@ -1021,23 +1058,23 @@ Display auth related authentication variables in shell friendly format. def st_auth(parser, args, thread_manager): (options, args) = parse_args(parser, args) - _opts = vars(options) - if options.verbose > 1: - if options.auth_version in ('1', '1.0'): - print('export ST_AUTH=%s' % sh_quote(options.auth)) - print('export ST_USER=%s' % sh_quote(options.user)) - print('export ST_KEY=%s' % sh_quote(options.key)) + if options['verbose'] > 1: + if options['auth_version'] in ('1', '1.0'): + print('export ST_AUTH=%s' % sh_quote(options['auth'])) + print('export ST_USER=%s' % sh_quote(options['user'])) + print('export ST_KEY=%s' % sh_quote(options['key'])) else: print('export OS_IDENTITY_API_VERSION=%s' % sh_quote( - options.auth_version)) - print('export OS_AUTH_VERSION=%s' % sh_quote(options.auth_version)) - print('export OS_AUTH_URL=%s' % sh_quote(options.auth)) - for k, v in sorted(_opts.items()): + options['auth_version'])) + print('export OS_AUTH_VERSION=%s' % sh_quote( + options['auth_version'])) + print('export OS_AUTH_URL=%s' % sh_quote(options['auth'])) + for k, v in sorted(options.items()): if v and k.startswith('os_') and \ k not in ('os_auth_url', 'os_options'): print('export %s=%s' % (k.upper(), sh_quote(v))) else: - conn = get_conn(_opts) + conn = get_conn(options) url, token = conn.get_auth() print('export OS_STORAGE_URL=%s' % sh_quote(url)) print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) @@ -1070,7 +1107,7 @@ Optional arguments: def st_tempurl(parser, args, thread_manager): - parser.add_option( + parser.add_argument( '--absolute', action='store_true', dest='absolute_expiry', default=False, help=("If present, seconds argument will be interpreted as a Unix " @@ -1094,81 +1131,88 @@ def st_tempurl(parser, args, thread_manager): 'tempurl specified, possibly an error' % method.upper()) url = generate_temp_url(path, seconds, key, method, - absolute=options.absolute_expiry) + absolute=options['absolute_expiry']) thread_manager.print_msg(url) +class HelpFormatter(argparse.HelpFormatter): + def _format_action_invocation(self, action): + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + metavar, = self._metavar_formatter(action, default)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s=ARGS, --long=ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s=%s' % (option_string, args_string)) + + return ', '.join(parts) + + # Back-port py3 methods + def _get_default_metavar_for_optional(self, action): + return action.dest.upper() + + def _get_default_metavar_for_positional(self, action): + return action.dest + + def parse_args(parser, args, enforce_requires=True): - if not args: - args = ['-h'] - (options, args) = parser.parse_args(args) + options, args = parser.parse_known_args(args or ['-h']) + options = vars(options) + if enforce_requires and (options['debug'] or options['info']): + logging.getLogger("swiftclient") + if options['debug']: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger('iso8601').setLevel(logging.WARNING) + client_logger_settings['redact_sensitive_headers'] = False + elif options['info']: + logging.basicConfig(level=logging.INFO) - if len(args) > 1 and args[1] == '--help': + if args and options['help']: _help = globals().get('st_%s_help' % args[0], "no help for %s" % args[0]) print(_help) exit() # Short circuit for tempurl, which doesn't need auth - if len(args) > 0 and args[0] == 'tempurl': + if args and args[0] == 'tempurl': return options, args - if options.auth_version == '3.0': - # tolerate sloppy auth_version - options.auth_version = '3' - - if (not (options.auth and options.user and options.key) - and options.auth_version != '3'): - # Use keystone auth if any of the old-style args are missing - options.auth_version = '2.0' - - # Use new-style args if old ones not present - if not options.auth and options.os_auth_url: - options.auth = options.os_auth_url - if not options.user and options.os_username: - options.user = options.os_username - if not options.key and options.os_password: - options.key = options.os_password - - # Specific OpenStack options - options.os_options = { - 'user_id': options.os_user_id, - 'user_domain_id': options.os_user_domain_id, - 'user_domain_name': options.os_user_domain_name, - 'tenant_id': options.os_tenant_id, - 'tenant_name': options.os_tenant_name, - 'project_id': options.os_project_id, - 'project_name': options.os_project_name, - 'project_domain_id': options.os_project_domain_id, - 'project_domain_name': options.os_project_domain_name, - 'service_type': options.os_service_type, - 'endpoint_type': options.os_endpoint_type, - 'auth_token': options.os_auth_token, - 'object_storage_url': options.os_storage_url, - 'region_name': options.os_region_name, - } + # Massage auth version; build out os_options subdict + process_options(options) if len(args) > 1 and args[0] == "capabilities": return options, args - if (options.os_options.get('object_storage_url') and - options.os_options.get('auth_token') and - (options.auth_version == '2.0' or options.auth_version == '3')): + if (options['os_options']['object_storage_url'] and + options['os_options']['auth_token']): return options, args if enforce_requires: - if options.auth_version == '3': - if not options.auth: + if options['auth_version'] == '3': + if not options['auth']: exit('Auth version 3 requires OS_AUTH_URL to be set or ' + 'overridden with --os-auth-url') - if not (options.user or options.os_user_id): + if not (options['user'] or options['os_user_id']): exit('Auth version 3 requires either OS_USERNAME or ' + 'OS_USER_ID to be set or overridden with ' + '--os-username or --os-user-id respectively.') - if not options.key: + if not options['key']: exit('Auth version 3 requires OS_PASSWORD to be set or ' + 'overridden with --os-password') - elif not (options.auth and options.user and options.key): + elif not (options['auth'] and options['user'] and options['key']): exit(''' Auth version 1.0 requires ST_AUTH, ST_USER, and ST_KEY environment variables to be set or overridden with -A, -U, or -K. @@ -1181,19 +1225,18 @@ adding "-V 2" is necessary for this.'''.strip('\n')) def main(arguments=None): - if arguments: - argv = arguments - else: - argv = sys_argv + argv = sys_argv if arguments is None else arguments argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] version = client_version - parser = OptionParser(version='python-swiftclient %s' % version, - usage=''' -usage: %prog [--version] [--help] [--os-help] [--snet] [--verbose] + parser = argparse.ArgumentParser( + add_help=False, formatter_class=HelpFormatter, usage=''' +%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] [--debug] [--info] [--quiet] [--auth <auth_url>] - [--auth-version <auth_version>] [--user <username>] + [--auth-version <auth_version> | + --os-identity-api-version <auth_version> ] + [--user <username>] [--key <api_key>] [--retries <num_retries>] [--os-username <auth-user-name>] [--os-password <auth-password>] [--os-user-id <auth-user-id>] @@ -1210,6 +1253,8 @@ usage: %prog [--version] [--help] [--os-help] [--snet] [--verbose] [--os-service-type <service-type>] [--os-endpoint-type <endpoint-type>] [--os-cacert <ca-certificate>] [--insecure] + [--os-cert <client-certificate-file>] + [--os-key <client-certificate-key-file>] [--no-ssl-compression] <subcommand> [--help] [<subcommand options>] @@ -1231,216 +1276,243 @@ Positional arguments: auth Display auth related environment variables. Examples: - %prog download --help + %(prog)s download --help - %prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key stat -v + %(prog)s -A https://auth.api.rackspacecloud.com/v1.0 \\ + -U user -K api_key stat -v - %prog --os-auth-url https://api.example.com/v2.0 --os-tenant-name tenant \\ + %(prog)s --os-auth-url https://api.example.com/v2.0 \\ + --os-tenant-name tenant \\ --os-username user --os-password password list - %prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ --os-project-name project1 --os-project-domain-name domain1 \\ --os-username user --os-user-domain-name domain1 \\ --os-password password list - %prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ --os-project-id 0123456789abcdef0123456789abcdef \\ --os-user-id abcdef0123456789abcdef0123456789 \\ --os-password password list - %prog --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ list - %prog list --lh + %(prog)s list --lh '''.strip('\n')) - parser.add_option('--os-help', action='store_true', dest='os_help', - help='Show OpenStack authentication options.') - parser.add_option('--os_help', action='store_true', help=SUPPRESS_HELP) - parser.add_option('-s', '--snet', action='store_true', dest='snet', - default=False, help='Use SERVICENET internal network.') - parser.add_option('-v', '--verbose', action='count', dest='verbose', - default=1, help='Print more info.') - parser.add_option('--debug', action='store_true', dest='debug', - default=False, help='Show the curl commands and results ' - 'of all http queries regardless of result status.') - parser.add_option('--info', action='store_true', dest='info', - default=False, help='Show the curl commands and results ' - 'of all http queries which return an error.') - parser.add_option('-q', '--quiet', action='store_const', dest='verbose', - const=0, default=1, help='Suppress status output.') - parser.add_option('-A', '--auth', dest='auth', - default=environ.get('ST_AUTH'), - help='URL for obtaining an auth token.') - parser.add_option('-V', '--auth-version', - dest='auth_version', - default=environ.get('ST_AUTH_VERSION', - (environ.get('OS_AUTH_VERSION', - '1.0'))), - type=str, - help='Specify a version for authentication. ' - 'Defaults to 1.0.') - parser.add_option('-U', '--user', dest='user', - default=environ.get('ST_USER'), - help='User name for obtaining an auth token.') - parser.add_option('-K', '--key', dest='key', - default=environ.get('ST_KEY'), - help='Key for obtaining an auth token.') - parser.add_option('-R', '--retries', type=int, default=5, dest='retries', - help='The number of times to retry a failed connection.') + parser.add_argument('--version', action='version', + version='python-swiftclient %s' % version) + parser.add_argument('-h', '--help', action='store_true') + + default_auth_version = '1.0' + for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): + try: + default_auth_version = environ[k] + break + except KeyError: + pass + + parser.add_argument('--os-help', action='store_true', dest='os_help', + help='Show OpenStack authentication options.') + parser.add_argument('--os_help', action='store_true', + help=argparse.SUPPRESS) + parser.add_argument('-s', '--snet', action='store_true', dest='snet', + default=False, help='Use SERVICENET internal network.') + parser.add_argument('-v', '--verbose', action='count', dest='verbose', + default=1, help='Print more info.') + parser.add_argument('--debug', action='store_true', dest='debug', + default=False, help='Show the curl commands and ' + 'results of all http queries regardless of result ' + 'status.') + parser.add_argument('--info', action='store_true', dest='info', + default=False, help='Show the curl commands and ' + 'results of all http queries which return an error.') + parser.add_argument('-q', '--quiet', action='store_const', dest='verbose', + const=0, default=1, help='Suppress status output.') + parser.add_argument('-A', '--auth', dest='auth', + default=environ.get('ST_AUTH'), + help='URL for obtaining an auth token.') + parser.add_argument('-V', '--auth-version', '--os-identity-api-version', + dest='auth_version', + default=default_auth_version, + type=str, + help='Specify a version for authentication. ' + 'Defaults to env[ST_AUTH_VERSION], ' + 'env[OS_AUTH_VERSION], ' + 'env[OS_IDENTITY_API_VERSION] or 1.0.') + parser.add_argument('-U', '--user', dest='user', + default=environ.get('ST_USER'), + help='User name for obtaining an auth token.') + parser.add_argument('-K', '--key', dest='key', + default=environ.get('ST_KEY'), + help='Key for obtaining an auth token.') + parser.add_argument('-R', '--retries', type=int, default=5, dest='retries', + help='The number of times to retry a failed ' + 'connection.') default_val = config_true_value(environ.get('SWIFTCLIENT_INSECURE')) - parser.add_option('--insecure', - action="store_true", dest="insecure", - default=default_val, - help='Allow swiftclient to access servers without ' - 'having to verify the SSL certificate. ' - 'Defaults to env[SWIFTCLIENT_INSECURE] ' - '(set to \'true\' to enable).') - parser.add_option('--no-ssl-compression', - action='store_false', dest='ssl_compression', - default=True, - help='This option is deprecated and not used anymore. ' - 'SSL compression should be disabled by default ' - 'by the system SSL library.') - - os_grp = OptionGroup(parser, "OpenStack authentication options") - os_grp.add_option('--os-username', - metavar='<auth-user-name>', - default=environ.get('OS_USERNAME'), - help='OpenStack username. Defaults to env[OS_USERNAME].') - os_grp.add_option('--os_username', - help=SUPPRESS_HELP) - os_grp.add_option('--os-user-id', - metavar='<auth-user-id>', - default=environ.get('OS_USER_ID'), - help='OpenStack user ID. ' - 'Defaults to env[OS_USER_ID].') - os_grp.add_option('--os_user_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-user-domain-id', - metavar='<auth-user-domain-id>', - default=environ.get('OS_USER_DOMAIN_ID'), - help='OpenStack user domain ID. ' - 'Defaults to env[OS_USER_DOMAIN_ID].') - os_grp.add_option('--os_user_domain_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-user-domain-name', - metavar='<auth-user-domain-name>', - default=environ.get('OS_USER_DOMAIN_NAME'), - help='OpenStack user domain name. ' - 'Defaults to env[OS_USER_DOMAIN_NAME].') - os_grp.add_option('--os_user_domain_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-password', - metavar='<auth-password>', - default=environ.get('OS_PASSWORD'), - help='OpenStack password. Defaults to env[OS_PASSWORD].') - os_grp.add_option('--os_password', - help=SUPPRESS_HELP) - os_grp.add_option('--os-tenant-id', - metavar='<auth-tenant-id>', - default=environ.get('OS_TENANT_ID'), - help='OpenStack tenant ID. ' - 'Defaults to env[OS_TENANT_ID].') - os_grp.add_option('--os_tenant_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-tenant-name', - metavar='<auth-tenant-name>', - default=environ.get('OS_TENANT_NAME'), - help='OpenStack tenant name. ' - 'Defaults to env[OS_TENANT_NAME].') - os_grp.add_option('--os_tenant_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-id', - metavar='<auth-project-id>', - default=environ.get('OS_PROJECT_ID'), - help='OpenStack project ID. ' - 'Defaults to env[OS_PROJECT_ID].') - os_grp.add_option('--os_project_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-name', - metavar='<auth-project-name>', - default=environ.get('OS_PROJECT_NAME'), - help='OpenStack project name. ' - 'Defaults to env[OS_PROJECT_NAME].') - os_grp.add_option('--os_project_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-domain-id', - metavar='<auth-project-domain-id>', - default=environ.get('OS_PROJECT_DOMAIN_ID'), - help='OpenStack project domain ID. ' - 'Defaults to env[OS_PROJECT_DOMAIN_ID].') - os_grp.add_option('--os_project_domain_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-domain-name', - metavar='<auth-project-domain-name>', - default=environ.get('OS_PROJECT_DOMAIN_NAME'), - help='OpenStack project domain name. ' - 'Defaults to env[OS_PROJECT_DOMAIN_NAME].') - os_grp.add_option('--os_project_domain_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-auth-url', - metavar='<auth-url>', - default=environ.get('OS_AUTH_URL'), - help='OpenStack auth URL. Defaults to env[OS_AUTH_URL].') - os_grp.add_option('--os_auth_url', - help=SUPPRESS_HELP) - os_grp.add_option('--os-auth-token', - metavar='<auth-token>', - default=environ.get('OS_AUTH_TOKEN'), - help='OpenStack token. Defaults to env[OS_AUTH_TOKEN]. ' - 'Used with --os-storage-url to bypass the ' - 'usual username/password authentication.') - os_grp.add_option('--os_auth_token', - help=SUPPRESS_HELP) - os_grp.add_option('--os-storage-url', - metavar='<storage-url>', - default=environ.get('OS_STORAGE_URL'), - help='OpenStack storage URL. ' - 'Defaults to env[OS_STORAGE_URL]. ' - 'Overrides the storage url returned during auth. ' - 'Will bypass authentication when used with ' - '--os-auth-token.') - os_grp.add_option('--os_storage_url', - help=SUPPRESS_HELP) - os_grp.add_option('--os-region-name', - metavar='<region-name>', - default=environ.get('OS_REGION_NAME'), - help='OpenStack region name. ' - 'Defaults to env[OS_REGION_NAME].') - os_grp.add_option('--os_region_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-service-type', - metavar='<service-type>', - default=environ.get('OS_SERVICE_TYPE'), - help='OpenStack Service type. ' - 'Defaults to env[OS_SERVICE_TYPE].') - os_grp.add_option('--os_service_type', - help=SUPPRESS_HELP) - os_grp.add_option('--os-endpoint-type', - metavar='<endpoint-type>', - default=environ.get('OS_ENDPOINT_TYPE'), - help='OpenStack Endpoint type. ' - 'Defaults to env[OS_ENDPOINT_TYPE].') - os_grp.add_option('--os_endpoint_type', - help=SUPPRESS_HELP) - os_grp.add_option('--os-cacert', - metavar='<ca-certificate>', - default=environ.get('OS_CACERT'), - help='Specify a CA bundle file to use in verifying a ' - 'TLS (https) server certificate. ' - 'Defaults to env[OS_CACERT].') - parser.disable_interspersed_args() - # call parse_args before adding os options group so that -h, --help will - # print a condensed help message without the os options - (options, args) = parse_args(parser, argv[1:], enforce_requires=False) - parser.add_option_group(os_grp) - if options.os_help: - # if openstack option help has been explicitly requested then force - # help message, now that os_options group has been added to parser - argv = ['-h'] - (options, args) = parse_args(parser, argv[1:], enforce_requires=False) - parser.enable_interspersed_args() + parser.add_argument('--insecure', + action="store_true", dest="insecure", + default=default_val, + help='Allow swiftclient to access servers without ' + 'having to verify the SSL certificate. ' + 'Defaults to env[SWIFTCLIENT_INSECURE] ' + '(set to \'true\' to enable).') + parser.add_argument('--no-ssl-compression', + action='store_false', dest='ssl_compression', + default=True, + help='This option is deprecated and not used anymore. ' + 'SSL compression should be disabled by default ' + 'by the system SSL library.') + + os_grp = parser.add_argument_group("OpenStack authentication options") + os_grp.add_argument('--os-username', + metavar='<auth-user-name>', + default=environ.get('OS_USERNAME'), + help='OpenStack username. Defaults to ' + 'env[OS_USERNAME].') + os_grp.add_argument('--os_username', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-user-id', + metavar='<auth-user-id>', + default=environ.get('OS_USER_ID'), + help='OpenStack user ID. ' + 'Defaults to env[OS_USER_ID].') + os_grp.add_argument('--os_user_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-user-domain-id', + metavar='<auth-user-domain-id>', + default=environ.get('OS_USER_DOMAIN_ID'), + help='OpenStack user domain ID. ' + 'Defaults to env[OS_USER_DOMAIN_ID].') + os_grp.add_argument('--os_user_domain_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-user-domain-name', + metavar='<auth-user-domain-name>', + default=environ.get('OS_USER_DOMAIN_NAME'), + help='OpenStack user domain name. ' + 'Defaults to env[OS_USER_DOMAIN_NAME].') + os_grp.add_argument('--os_user_domain_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-password', + metavar='<auth-password>', + default=environ.get('OS_PASSWORD'), + help='OpenStack password. Defaults to ' + 'env[OS_PASSWORD].') + os_grp.add_argument('--os_password', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-tenant-id', + metavar='<auth-tenant-id>', + default=environ.get('OS_TENANT_ID'), + help='OpenStack tenant ID. ' + 'Defaults to env[OS_TENANT_ID].') + os_grp.add_argument('--os_tenant_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-tenant-name', + metavar='<auth-tenant-name>', + default=environ.get('OS_TENANT_NAME'), + help='OpenStack tenant name. ' + 'Defaults to env[OS_TENANT_NAME].') + os_grp.add_argument('--os_tenant_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-id', + metavar='<auth-project-id>', + default=environ.get('OS_PROJECT_ID'), + help='OpenStack project ID. ' + 'Defaults to env[OS_PROJECT_ID].') + os_grp.add_argument('--os_project_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-name', + metavar='<auth-project-name>', + default=environ.get('OS_PROJECT_NAME'), + help='OpenStack project name. ' + 'Defaults to env[OS_PROJECT_NAME].') + os_grp.add_argument('--os_project_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-domain-id', + metavar='<auth-project-domain-id>', + default=environ.get('OS_PROJECT_DOMAIN_ID'), + help='OpenStack project domain ID. ' + 'Defaults to env[OS_PROJECT_DOMAIN_ID].') + os_grp.add_argument('--os_project_domain_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-domain-name', + metavar='<auth-project-domain-name>', + default=environ.get('OS_PROJECT_DOMAIN_NAME'), + help='OpenStack project domain name. ' + 'Defaults to env[OS_PROJECT_DOMAIN_NAME].') + os_grp.add_argument('--os_project_domain_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-auth-url', + metavar='<auth-url>', + default=environ.get('OS_AUTH_URL'), + help='OpenStack auth URL. Defaults to ' + 'env[OS_AUTH_URL].') + os_grp.add_argument('--os_auth_url', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-auth-token', + metavar='<auth-token>', + default=environ.get('OS_AUTH_TOKEN'), + help='OpenStack token. Defaults to ' + 'env[OS_AUTH_TOKEN]. Used with --os-storage-url ' + 'to bypass the usual username/password ' + 'authentication.') + os_grp.add_argument('--os_auth_token', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-storage-url', + metavar='<storage-url>', + default=environ.get('OS_STORAGE_URL'), + help='OpenStack storage URL. ' + 'Defaults to env[OS_STORAGE_URL]. ' + 'Overrides the storage url returned during auth. ' + 'Will bypass authentication when used with ' + '--os-auth-token.') + os_grp.add_argument('--os_storage_url', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-region-name', + metavar='<region-name>', + default=environ.get('OS_REGION_NAME'), + help='OpenStack region name. ' + 'Defaults to env[OS_REGION_NAME].') + os_grp.add_argument('--os_region_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-service-type', + metavar='<service-type>', + default=environ.get('OS_SERVICE_TYPE'), + help='OpenStack Service type. ' + 'Defaults to env[OS_SERVICE_TYPE].') + os_grp.add_argument('--os_service_type', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-endpoint-type', + metavar='<endpoint-type>', + default=environ.get('OS_ENDPOINT_TYPE'), + help='OpenStack Endpoint type. ' + 'Defaults to env[OS_ENDPOINT_TYPE].') + os_grp.add_argument('--os_endpoint_type', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-cacert', + metavar='<ca-certificate>', + default=environ.get('OS_CACERT'), + help='Specify a CA bundle file to use in verifying a ' + 'TLS (https) server certificate. ' + 'Defaults to env[OS_CACERT].') + os_grp.add_argument('--os-cert', + metavar='<client-certificate-file>', + default=environ.get('OS_CERT'), + help='Specify a client certificate file (for client ' + 'auth). Defaults to env[OS_CERT].') + os_grp.add_argument('--os-key', + metavar='<client-certificate-key-file>', + default=environ.get('OS_KEY'), + help='Specify a client certificate key file (for ' + 'client auth). Defaults to env[OS_KEY].') + options, args = parse_args(parser, argv[1:], enforce_requires=False) + + if options['help'] or options['os_help']: + if options['help']: + parser._action_groups.pop() + parser.print_help() + exit() if not args or args[0] not in commands: parser.print_usage() @@ -1450,20 +1522,17 @@ Examples: signal.signal(signal.SIGINT, immediate_exit) - if options.debug or options.info: - logging.getLogger("swiftclient") - if options.debug: - logging.basicConfig(level=logging.DEBUG) - logging.getLogger('iso8601').setLevel(logging.WARNING) - elif options.info: - logging.basicConfig(level=logging.INFO) - with OutputManager() as output: - parser.usage = globals()['st_%s_help' % args[0]] try: globals()['st_%s' % args[0]](parser, argv[1:], output) - except (ClientException, RequestException, socket.error) as err: + except ClientException as err: + output.error(str(err)) + trans_id = (err.http_response_headers or {}).get('X-Trans-Id') + if trans_id: + output.error("Failed Transaction ID: %s", + parse_header_string(trans_id)) + except (RequestException, socket.error) as err: output.error(str(err)) if output.get_error_count() > 0: diff --git a/test-requirements.txt b/test-requirements.txt index 7f7e405..0a81398 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,10 +1,7 @@ hacking>=0.10.0,<0.11 coverage>=3.6 -discover mock>=1.2 oslosphinx -python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 testrepository>=0.0.18 -testtools>=0.9.34 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 5f9e271..7a77c07 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -14,7 +14,7 @@ # limitations under the License. import os -import testtools +import unittest import time from io import BytesIO @@ -23,7 +23,7 @@ from six.moves import configparser import swiftclient -class TestFunctional(testtools.TestCase): +class TestFunctional(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestFunctional, self).__init__(*args, **kwargs) diff --git a/tests/unit/test_command_helpers.py b/tests/unit/test_command_helpers.py index d9d7efa..24684ae 100644 --- a/tests/unit/test_command_helpers.py +++ b/tests/unit/test_command_helpers.py @@ -15,13 +15,13 @@ import mock from six import StringIO -import testtools +import unittest from swiftclient import command_helpers as h from swiftclient.multithreading import OutputManager -class TestStatHelpers(testtools.TestCase): +class TestStatHelpers(unittest.TestCase): def setUp(self): super(TestStatHelpers, self).setUp() diff --git a/tests/unit/test_multithreading.py b/tests/unit/test_multithreading.py index 76758b6..8944d48 100644 --- a/tests/unit/test_multithreading.py +++ b/tests/unit/test_multithreading.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import sys -import testtools +import unittest import threading import six @@ -25,7 +25,7 @@ from swiftclient import multithreading as mt from .utils import CaptureStream -class ThreadTestCase(testtools.TestCase): +class ThreadTestCase(unittest.TestCase): def setUp(self): super(ThreadTestCase, self).setUp() self.got_items = Queue() @@ -163,7 +163,7 @@ class TestConnectionThreadPoolExecutor(ThreadTestCase): ) -class TestOutputManager(testtools.TestCase): +class TestOutputManager(unittest.TestCase): def test_instantiation(self): output_manager = mt.OutputManager() diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 64039ad..cce8c7a 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -18,7 +18,7 @@ import mock import os import six import tempfile -import testtools +import unittest import time from concurrent.futures import Future @@ -49,7 +49,7 @@ else: import builtins -class TestSwiftPostObject(testtools.TestCase): +class TestSwiftPostObject(unittest.TestCase): def setUp(self): super(TestSwiftPostObject, self).setUp() @@ -69,7 +69,7 @@ class TestSwiftPostObject(testtools.TestCase): self.assertRaises(SwiftError, self.spo, 1) -class TestSwiftReader(testtools.TestCase): +class TestSwiftReader(unittest.TestCase): def setUp(self): super(TestSwiftReader, self).setUp() @@ -152,25 +152,7 @@ class TestSwiftReader(testtools.TestCase): '97ac82a5b825239e782d0339e2d7b910') -class _TestServiceBase(testtools.TestCase): - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if not m: - m = '{0} != {1}'.format(a, b) - - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertIsInstance(a, dict, - 'First argument is not a dictionary') - self.assertIsInstance(b, dict, - 'Second argument is not a dictionary') - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertIn(k, b, m) - self.assertEqual(b[k], v, m) - +class _TestServiceBase(unittest.TestCase): def _get_mock_connection(self, attempts=2): m = Mock(spec=Connection) type(m).attempts = PropertyMock(return_value=attempts) @@ -223,8 +205,8 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) - self._assertDictEqual(expected_r, r) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) def test_delete_segment_exception(self): mock_q = Queue() @@ -246,8 +228,8 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) - self._assertDictEqual(expected_r, r) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) @@ -268,7 +250,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_object_exception(self): mock_q = Queue() @@ -294,7 +276,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) @@ -321,7 +303,7 @@ class TestServiceDelete(_TestServiceBase): query_string='multipart-manifest=delete', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_object_dlo_support(self): mock_q = Queue() @@ -352,7 +334,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn, 'test_c', 'test_o', self.opts, mock_q ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) expected = [ mock.call('test_c', 'test_o', query_string=None, response_dict={}), mock.call('manifest_c', 'test_seg_1', response_dict={}), @@ -372,7 +354,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_empty_container_exception(self): mock_conn = self._get_mock_connection() @@ -394,13 +376,13 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) -class TestSwiftError(testtools.TestCase): +class TestSwiftError(unittest.TestCase): def test_is_exception(self): se = SwiftError(5) @@ -430,7 +412,7 @@ class TestSwiftError(testtools.TestCase): self.assertEqual(str(se), '5 container:con object:obj segment:seg') -class TestServiceUtils(testtools.TestCase): +class TestServiceUtils(unittest.TestCase): def setUp(self): super(TestServiceUtils, self).setUp() @@ -525,7 +507,7 @@ class TestServiceUtils(testtools.TestCase): mock_headers) -class TestSwiftUploadObject(testtools.TestCase): +class TestSwiftUploadObject(unittest.TestCase): def setUp(self): self.suo = swiftclient.service.SwiftUploadObject @@ -614,7 +596,7 @@ class TestServiceList(_TestServiceBase): SwiftService._list_account_job( mock_conn, self.opts, mock_q ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) @@ -635,7 +617,7 @@ class TestServiceList(_TestServiceBase): SwiftService._list_account_job( mock_conn, long_opts, mock_q ) - self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_account_exception(self): @@ -657,7 +639,7 @@ class TestServiceList(_TestServiceBase): mock_conn.get_account.assert_called_once_with( marker='', prefix=None ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_container(self): @@ -680,7 +662,7 @@ class TestServiceList(_TestServiceBase): SwiftService._list_container_job( mock_conn, 'test_c', self.opts, mock_q ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) @@ -702,7 +684,42 @@ class TestServiceList(_TestServiceBase): SwiftService._list_container_job( mock_conn, 'test_c', long_opts, mock_q ) - self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container_marker(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + + get_container_returns = [ + (None, [{'name': 'b'}, {'name': 'c'}]), + (None, []) + ] + mock_get_cont = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock_get_cont + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'b'}, {'name': 'c'}], + 'marker': 'b' + }) + + _opts = self.opts.copy() + _opts['marker'] = 'b' + SwiftService._list_container_job(mock_conn, 'test_c', _opts, mock_q) + + # This does not test if the marker is propagated, because we always + # get the final call to the get_container with the final item 'c', + # even if marker wasn't set. This test just makes sure the whole + # stack works in a sane way. + mock_kw = mock_get_cont.call_args[1] + self.assertEqual(mock_kw['marker'], 'c') + + # This tests that the lower levels get the marker delivered. + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) def test_list_container_exception(self): @@ -726,7 +743,7 @@ class TestServiceList(_TestServiceBase): mock_conn.get_container.assert_called_once_with( 'test_c', marker='', delimiter='', prefix=None ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) @mock.patch('swiftclient.service.get_conn') @@ -805,7 +822,7 @@ class TestServiceList(_TestServiceBase): self.assertEqual(observed_listing, expected_listing) -class TestService(testtools.TestCase): +class TestService(unittest.TestCase): def test_upload_with_bad_segment_size(self): for bad in ('ten', '1234X', '100.3'): @@ -913,7 +930,7 @@ class TestServiceUpload(_TestServiceBase): self.assertEqual(r['path'], f.name) del r['path'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'テスト/dummy.dat', '', @@ -960,14 +977,15 @@ class TestServiceUpload(_TestServiceBase): options={'segment_container': None, 'checksum': True}) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c_segments', - 'test_s_1', - mock.ANY, - content_length=10, - response_dict={}) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) @@ -1006,11 +1024,12 @@ class TestServiceUpload(_TestServiceBase): self.assertIsNone(r.get('error')) self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c_segments', - 'test_s_1', - mock.ANY, - content_length=10, - response_dict={}) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) contents = mock_conn.put_object.call_args[0][2] # Check that md5sum is not calculated. self.assertEqual(contents.get_md5sum(), '') @@ -1046,11 +1065,12 @@ class TestServiceUpload(_TestServiceBase): self.assertIn('md5 mismatch', str(r.get('error'))) self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c_segments', - 'test_s_1', - mock.ANY, - content_length=10, - response_dict={}) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest()) @@ -1098,7 +1118,7 @@ class TestServiceUpload(_TestServiceBase): self.assertEqual(r['path'], f.name) del r['path'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, @@ -1155,7 +1175,7 @@ class TestServiceUpload(_TestServiceBase): self.assertEqual(mtime, expected_mtime) del r['headers']['x-object-meta-mtime'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, @@ -1340,6 +1360,168 @@ class TestServiceUpload(_TestServiceBase): ] mock_conn.get_container.assert_has_calls(expected) + def test_make_upload_objects(self): + check_names_pseudo_to_expected = { + (('/absolute/file/path',), ''): ['absolute/file/path'], + (('relative/file/path',), ''): ['relative/file/path'], + (('/absolute/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/absolute/file/path'], + (('/absolute/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/absolute/file/path'], + (('relative/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/relative/file/path'], + (('relative/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/relative/file/path'], + } + errors = [] + for (filenames, pseudo_folder), expected in \ + check_names_pseudo_to_expected.items(): + actual = SwiftService._make_upload_objects( + filenames, pseudo_folder=pseudo_folder) + try: + self.assertEqual(expected, [o.object_name for o in actual]) + except AssertionError as e: + msg = 'given (%r, %r) expected %r, got %s' % ( + filenames, pseudo_folder, expected, e) + errors.append(msg) + self.assertFalse(errors, "\nERRORS:\n%s" % '\n'.join(errors)) + + def test_create_dir_marker_job_unchanged(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'application/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.234000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.getmtime', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + path='test', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': 'test', + 'headers': {'x-object-meta-mtime': '1.234000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_unchanged_old_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_overwrites_bad_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/plain', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_missing(self): + mock_conn = mock.Mock() + mock_conn.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + class TestServiceDownload(_TestServiceBase): @@ -1533,7 +1715,7 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_with_mtime(self): mock_conn = self._get_mock_connection() @@ -1579,7 +1761,7 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_bad_mtime(self): mock_conn = self._get_mock_connection() @@ -1624,7 +1806,7 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() @@ -1644,19 +1826,18 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} mock_conn.get_object.return_value = header, self._readbody() - resp = service._download_object_job(mock_conn, - 'c', - 'test', - self.opts) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'test', + self.opts) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1664,8 +1845,23 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual(resp['object'], 'test') self.assertEqual(resp['path'], 'test') - def test_download_with_output_dir(self): + @mock.patch('swiftclient.service.interruptable_as_completed') + @mock.patch('swiftclient.service.SwiftService._download_container') + @mock.patch('swiftclient.service.SwiftService._download_object_job') + def test_download_with_objects_empty(self, mock_down_obj, + mock_down_cont, mock_as_comp): + fake_future = Future() + fake_future.set_result(1) + mock_as_comp.return_value = [fake_future] service = SwiftService() + next(service.download('c', [], self.opts), None) + mock_down_obj.assert_not_called() + mock_down_cont.assert_not_called() + + next(service.download('c', options=self.opts), None) + self.assertEqual(True, mock_down_cont.called) + + def test_download_with_output_dir(self): with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1673,10 +1869,10 @@ class TestServiceDownload(_TestServiceBase): options = self.opts.copy() options['out_directory'] = 'temp_dir' - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1685,7 +1881,6 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual(resp['path'], 'temp_dir/example/test') def test_download_with_remove_prefix(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1694,10 +1889,10 @@ class TestServiceDownload(_TestServiceBase): options = self.opts.copy() options['prefix'] = 'example/' options['remove_prefix'] = True - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1706,7 +1901,6 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual(resp['path'], 'test') def test_download_with_remove_prefix_and_remove_slashes(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1715,10 +1909,10 @@ class TestServiceDownload(_TestServiceBase): options = self.opts.copy() options['prefix'] = 'example' options['remove_prefix'] = True - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1727,7 +1921,6 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual(resp['path'], 'test') def test_download_with_output_dir_and_remove_prefix(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1737,10 +1930,10 @@ class TestServiceDownload(_TestServiceBase): options['prefix'] = 'example' options['out_directory'] = 'new/dir' options['remove_prefix'] = True - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1788,7 +1981,7 @@ class TestServiceDownload(_TestServiceBase): 'header': {}, 'yes_all': False, 'skip_identical': True}) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.call_count, 1) mock_conn.get_object.assert_called_with( @@ -1850,7 +2043,7 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual("Large object is identical", err.msg) self.assertEqual(304, err.http_status) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.call_count, 1) mock_conn.get_object.assert_called_with( @@ -1933,7 +2126,7 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual("Large object is identical", err.msg) self.assertEqual(304, err.http_status) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', @@ -1999,7 +2192,7 @@ class TestServiceDownload(_TestServiceBase): obj='test_o', options=options) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_container.mock_calls, [ mock.call('test_c_segments', @@ -2090,7 +2283,7 @@ class TestServiceDownload(_TestServiceBase): obj='test_o', options=options) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', @@ -2106,3 +2299,48 @@ class TestServiceDownload(_TestServiceBase): resp_chunk_size=65536, headers={'If-None-Match': on_disk_md5}, response_dict={})]) + + +class TestServicePost(_TestServiceBase): + + def setUp(self): + super(TestServicePost, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + + @mock.patch('swiftclient.service.MultiThreadingManager') + @mock.patch('swiftclient.service.ResultsIterator') + def test_object_post(self, res_iter, thread_manager): + """ + Check post method translates strings and objects to _post_object_job + calls correctly + """ + tm_instance = Mock() + thread_manager.return_value = tm_instance + + self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) + spo = swiftclient.service.SwiftPostObject( + "test_spo", + {'meta': ["meta1:test2"], "header": ["hdr1:test2"]}) + + SwiftService().post('test_c', ['test_o', spo], self.opts) + + calls = [ + mock.call( + SwiftService._post_object_job, 'test_c', 'test_o', + { + "X-Object-Meta-Meta1": "test1", + "Hdr1": "test1"}, + {}), + mock.call( + SwiftService._post_object_job, 'test_c', 'test_spo', + { + "X-Object-Meta-Meta1": "test2", + "Hdr1": "test2"}, + {}), + ] + tm_instance.object_uu_pool.submit.assert_has_calls(calls) + self.assertEqual( + tm_instance.object_uu_pool.submit.call_count, len(calls)) + + res_iter.assert_called_with( + [tm_instance.object_uu_pool.submit()] * len(calls)) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 00546f6..d82def6 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -16,12 +16,12 @@ from __future__ import unicode_literals from genericpath import getmtime import hashlib +import logging import mock import os import tempfile -import testtools +import unittest import textwrap -from testtools import ExpectedException import six @@ -62,16 +62,19 @@ def _make_args(cmd, opts, os_opts, separator='-', flags=None, cmd_args=None): args = [""] flags = flags or [] for k, v in opts.items(): - arg = "--" + k.replace("_", "-") - args = args + [arg, v] + args.append("--" + k.replace("_", "-")) + if v is not None: + args.append(v) for k, v in os_opts.items(): - arg = "--os" + separator + k.replace("_", separator) - args = args + [arg, v] + args.append("--os" + separator + k.replace("_", separator)) + if v is not None: + args.append(v) for flag in flags: args.append('--%s' % flag) - args = args + [cmd] + if cmd: + args.append(cmd) if cmd_args: - args = args + cmd_args + args.extend(cmd_args) return args @@ -105,7 +108,7 @@ def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): @mock.patch.dict(os.environ, mocked_os_environ) -class TestShell(testtools.TestCase): +class TestShell(unittest.TestCase): def setUp(self): super(TestShell, self).setUp() tmpfile = tempfile.NamedTemporaryFile(delete=False) @@ -502,8 +505,8 @@ class TestShell(testtools.TestCase): response_dict={}) connection.return_value.put_object.assert_called_with( - 'container/pseudo-folder/nested', - self.tmpfile.lstrip('/'), + 'container', + 'pseudo-folder/nested' + self.tmpfile, mock.ANY, content_length=0, headers={'x-object-meta-mtime': mock.ANY, @@ -548,12 +551,40 @@ class TestShell(testtools.TestCase): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + # upload in segments to pseudo-folder (via <container> param) + connection.reset_mock() + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container/pseudo-folder/nested", + self.tmpfile, "-S", "10", "--use-slo"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': 'one'}, + response_dict={})] + connection.return_value.put_container.assert_has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + headers={ + 'x-object-meta-mtime': mock.ANY, + 'x-static-large-object': 'true' + }, + query_string='multipart-manifest=put', + response_dict={}) + @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): argv = ["", "upload", "container", self.tmpfile] upload.return_value = [ {"success": False, "headers": {}, + "container": 'container', "action": 'create_container', "error": swiftclient.ClientException( 'Container PUT failed', @@ -660,10 +691,10 @@ class TestShell(testtools.TestCase): [None, []] ] connection.return_value.put_object.return_value = EMPTY_ETAG - swiftclient.shell.main(argv) # create the delete_object child mock here in attempt to fix # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) connection.return_value.put_object.assert_called_with( 'container', self.tmpfile.lstrip('/'), @@ -734,6 +765,37 @@ class TestShell(testtools.TestCase): @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', lambda *a: False) @mock.patch('swiftclient.service.Connection') + def test_delete_bad_threads(self, mock_connection): + mock_connection.return_value.get_container.return_value = (None, []) + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) + @mock.patch('swiftclient.service.Connection') def test_delete_account(self, connection): connection.return_value.get_account.side_effect = [ [None, [{'name': 'container'}, {'name': 'container2'}]], @@ -750,6 +812,7 @@ class TestShell(testtools.TestCase): connection.return_value.attempts = 0 argv = ["", "delete", "--all"] connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None swiftclient.shell.main(argv) connection.return_value.delete_object.assert_has_calls([ mock.call('container', 'object', query_string=None, @@ -791,22 +854,31 @@ class TestShell(testtools.TestCase): b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') swiftclient.shell.main(argv) self.assertEqual( - connection.return_value.post_account.mock_calls, [ - mock.call(query_string='bulk-delete', - data=b'/container/object\n/container/obj%C3%A9ct2\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b'/container/object3\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b'/container2/object\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={})]) + 3, len(connection.return_value.post_account.mock_calls), + 'Expected 3 calls but found\n%r' + % connection.return_value.post_account.mock_calls) + # POSTs for same container are made in parallel so expect any order + for expected in [ + mock.call(query_string='bulk-delete', + data=b'/container/object\n/container/obj%C3%A9ct2\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b'/container/object3\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]: + self.assertIn(expected, + connection.return_value.post_account.mock_calls[:2]) + # POSTs for different containers are made sequentially so expect order + self.assertEqual( + mock.call(query_string='bulk-delete', + data=b'/container2/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + connection.return_value.post_account.mock_calls[2]) self.assertEqual( connection.return_value.delete_container.mock_calls, [ mock.call('container', response_dict={}), @@ -1036,13 +1108,42 @@ class TestShell(testtools.TestCase): def test_post_account_bad_auth(self, connection): argv = ["", "post"] connection.return_value.post_account.side_effect = \ - swiftclient.ClientException('bad auth') + swiftclient.ClientException( + 'bad auth', http_response_headers={'X-Trans-Id': 'trans_id'}) with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) - self.assertEqual(output.err, 'bad auth\n') + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: trans_id\n') + + # do it again with a unicode token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': 'non\u2011utf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\n' + 'Failed Transaction ID: non\u2011utf8\n') + + # do it again with a wonky token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': b'non\xffutf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: non%FFutf8\n') @mock.patch('swiftclient.service.Connection') def test_post_account_not_found(self, connection): @@ -1051,7 +1152,7 @@ class TestShell(testtools.TestCase): swiftclient.ClientException('test', http_status=404) with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'Account not found\n') @@ -1070,7 +1171,7 @@ class TestShell(testtools.TestCase): swiftclient.ClientException('bad auth') with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1088,7 +1189,7 @@ class TestShell(testtools.TestCase): argv = ["", "post", "conta/iner"] with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertTrue(output.err != '') self.assertTrue(output.err.startswith('WARNING: / in')) @@ -1128,7 +1229,7 @@ class TestShell(testtools.TestCase): swiftclient.ClientException("bad auth") with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1137,7 +1238,7 @@ class TestShell(testtools.TestCase): argv = ["", "post", "container", "object", "bad_arg"] with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertTrue(output.err != '') @@ -1198,49 +1299,49 @@ class TestShell(testtools.TestCase): _check_expected(mock_swift, 12345) with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): # Test invalid states argv = ["", "upload", "-S", "1234X", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "K1234", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "K", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") def test_negative_upload_segment_size(self): with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): - argv = ["", "upload", "-S", "-40K", "container", "object"] + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40K", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): - argv = ["", "upload", "-S", "-40M", "container", "object"] + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40M", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): - argv = ["", "upload", "-S", "-40G", "container", "object"] + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40G", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() -class TestSubcommandHelp(testtools.TestCase): +class TestSubcommandHelp(unittest.TestCase): def test_subcommand_help(self): for command in swiftclient.shell.commands: @@ -1260,7 +1361,39 @@ class TestSubcommandHelp(testtools.TestCase): self.assertEqual(out.strip('\n'), expected) -class TestBase(testtools.TestCase): +@mock.patch.dict(os.environ, mocked_os_environ) +class TestDebugAndInfoOptions(unittest.TestCase): + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_option_after_posarg(self, connection, mock_logging): + argv = ["", "stat", "--info"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.INFO) + + argv = ["", "stat", "--debug"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.DEBUG) + + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_debug_trumps_info(self, connection, mock_logging): + argv_scenarios = (["", "stat", "--info", "--debug"], + ["", "stat", "--debug", "--info"], + ["", "--info", "stat", "--debug"], + ["", "--debug", "stat", "--info"], + ["", "--info", "--debug", "stat"], + ["", "--debug", "--info", "stat"]) + for argv in argv_scenarios: + mock_logging.reset_mock() + swiftclient.shell.main(argv) + try: + mock_logging.assert_called_once_with(level=logging.DEBUG) + except AssertionError: + self.fail('Unexpected call(s) %r for args %r' + % (mock_logging.call_args_list, argv)) + + +class TestBase(unittest.TestCase): """ Provide some common methods to subclasses """ @@ -1291,29 +1424,32 @@ class TestParsing(TestBase): result[0], result[1] = swiftclient.shell.parse_args(parser, args) return fake_command - def _verify_opts(self, actual_opts, opts, os_opts={}, os_opts_dict={}): + def _verify_opts(self, actual_opts, expected_opts, expected_os_opts=None, + expected_os_opts_dict=None): """ Check parsed options are correct. - :param opts: v1 style options. - :param os_opts: openstack style options. - :param os_opts_dict: openstack options that should be found in the - os_options dict. + :param expected_opts: v1 style options. + :param expected_os_opts: openstack style options. + :param expected_os_opts_dict: openstack options that should be found in + the os_options dict. """ + expected_os_opts = expected_os_opts or {} + expected_os_opts_dict = expected_os_opts_dict or {} # check the expected opts are set - for key, v in opts.items(): - actual = getattr(actual_opts, key) + for key, v in expected_opts.items(): + actual = actual_opts.get(key) self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % (v, key, actual)) - for key, v in os_opts.items(): - actual = getattr(actual_opts, "os_" + key) + for key, v in expected_os_opts.items(): + actual = actual_opts.get("os_" + key) self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % (v, key, actual)) # check the os_options dict values are set - self.assertTrue(hasattr(actual_opts, 'os_options')) - actual_os_opts_dict = getattr(actual_opts, 'os_options') + self.assertIn('os_options', actual_opts) + actual_os_opts_dict = actual_opts['os_options'] expected_os_opts_keys = ['project_name', 'region_name', 'tenant_name', 'user_domain_name', 'endpoint_type', @@ -1327,8 +1463,8 @@ class TestParsing(TestBase): if key == 'object_storage_url': # exceptions to the pattern... cli_key = 'storage_url' - if cli_key in os_opts_dict: - expect = os_opts_dict[cli_key] + if cli_key in expected_os_opts_dict: + expect = expected_os_opts_dict[cli_key] else: expect = None actual = actual_os_opts_dict[key] @@ -1342,8 +1478,8 @@ class TestParsing(TestBase): ('os_auth_url', 'auth'), ('os_password', 'key')] for pair in equivalents: - self.assertEqual(getattr(actual_opts, pair[0]), - getattr(actual_opts, pair[1])) + self.assertEqual(actual_opts.get(pair[0]), + actual_opts.get(pair[1])) def test_minimum_required_args_v3(self): opts = {"auth_version": "3"} @@ -1386,6 +1522,132 @@ class TestParsing(TestBase): swiftclient.shell.main(args) self._verify_opts(result[0], opts, os_opts, os_opts_dict) + def test_sloppy_versions(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "identity-api-version": "3.0"} + + # check os_identity_api_version=3.0 is mapped to auth_version=3 + args = _make_args("stat", {}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} # NB: not '3.0' + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check os_identity_api_version=2 is mapped to auth_version=2.0 + # A somewhat contrived scenario - we need to pass in the v1 style opts + # to prevent auth version defaulting to 2.0 due to lack of v1 style + # options. That way we can actually verify that the sloppy 2 was + # interpreted and mapped to 2.0 + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v2.0", + "identity-api-version": "2"} + opts = {"key": "secret", + "user": "user", + "auth": "http://example.com:5000/v2.0"} + args = _make_args("stat", opts, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} # NB: not '2' + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v2.0"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + def test_os_identity_api_version(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "identity-api-version": "3"} + + # check os_identity_api_version is sufficient in place of auth_version + args = _make_args("stat", {}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check again using environment variables + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that last of auth-version, os-identity-api-version is preferred + args = _make_args("stat", {}, os_opts, '-') + ['--auth-version', '2.0'] + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # now put auth_version ahead of os-identity-api-version + args = _make_args("stat", {"auth_version": "2.0"}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that OS_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'OS_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_AUTH_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0', 'OS_AUTH_VERSION': '3'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + def test_args_v3(self): opts = {"auth_version": "3"} os_opts = {"password": "secret", @@ -1537,17 +1799,17 @@ class TestParsing(TestBase): def test_help(self): # --help returns condensed help message - opts = {"help": ""} + opts = {"help": None} os_opts = {} - args = _make_args("stat", opts, os_opts) + args = _make_args(None, opts, os_opts) with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) self.assertTrue(out.find('[--key <api_key>]') > 0) self.assertEqual(-1, out.find('--os-username=<auth-user-name>')) # --help returns condensed help message, overrides --os-help - opts = {"help": ""} - os_opts = {"help": ""} + opts = {"help": None} + os_opts = {"help": None} args = _make_args("", opts, os_opts) with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) @@ -1556,8 +1818,8 @@ class TestParsing(TestBase): # --os-password, --os-username and --os-auth_url should be ignored # because --help overrides it - opts = {"help": ""} - os_opts = {"help": "", + opts = {"help": None} + os_opts = {"help": None, "password": "secret", "username": "user", "auth_url": "http://example.com:5000/v3"} @@ -1593,7 +1855,9 @@ class TestKeystoneOptions(MockHttpTest): 'project-id': 'projectid', 'project-domain-id': 'projectdomainid', 'project-domain-name': 'projectdomain', - 'cacert': 'foo'} + 'cacert': 'foo', + 'cert': 'minnie', + 'key': 'mickey'} catalog_opts = {'service-type': 'my-object-store', 'endpoint-type': 'public', 'region-name': 'my-region'} @@ -1715,6 +1979,13 @@ class TestKeystoneOptions(MockHttpTest): self._test_options(opts, os_opts, flags=self.flags) opts = {} + self.defaults['auth-version'] = '3' + self._test_options(opts, os_opts, flags=self.flags) + + for o in ('user-domain-name', 'user-domain-id', + 'project-domain-name', 'project-domain-id'): + os_opts.pop(o) + self.defaults['auth-version'] = '2.0' self._test_options(opts, os_opts, flags=self.flags) def test_catalog_options_and_flags_not_required_v3(self): @@ -1765,6 +2036,28 @@ class TestKeystoneOptions(MockHttpTest): os_opts = self._build_os_opts(keys) self._test_options(opts, os_opts) + # ...except when it should be 3 + self.defaults['auth-version'] = '3' + keys = ('username', 'user-domain-name', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'user-domain-id', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'project-domain-name', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'project-domain-id', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + def test_url_and_token_provided_on_command_line(self): endpoint = 'http://alternate.com:8080/v1/AUTH_another' token = 'alternate_auth_token' @@ -2039,6 +2332,38 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): return status return on_request + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) + @mock.patch('swiftclient.service.Connection') + def test_upload_bad_threads(self, mock_connection): + mock_connection.return_value.put_object.return_value = EMPTY_ETAG + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'upload', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'upload', + cmd_args=[self.cont, self.obj, '--leave-segments'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--segment-threads", "-1"]) + check_bad(["--segment-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--segment-threads", "1"]) + def test_upload_with_read_write_access(self): req_handler = self._fake_cross_account_auth(True, True) fake_conn = self.fake_http_connection(403, 403, @@ -2120,6 +2445,53 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): % self.cont self.assertEqual(expected_err, out.err.strip()) + def test_segment_upload_with_write_only_access_segments_container(self): + fake_conn = self.fake_http_connection( + 403, # PUT c1 + # HEAD c1 to get storage policy + StubResponse(200, headers={'X-Storage-Policy': 'foo'}), + 403, # PUT c1_segments + 201, # PUT c1_segments/...00 + 201, # PUT c1_segments/...01 + 201, # PUT c1/... + ) + + args, env = self._make_cmd('upload', + cmd_args=[self.cont, self.obj, + '--leave-segments', + '--segment-size=10']) + with mock.patch('swiftclient.client._import_keystone_client', + self.fake_ks_import): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + segment_time = getmtime(self.obj) + segment_path_0 = '%s_segments%s/%f/20/10/00000000' % ( + self.cont_path, self.obj, segment_time) + segment_path_1 = '%s_segments%s/%f/20/10/00000001' % ( + self.cont_path, self.obj, segment_time) + # Note that the order of segment PUTs cannot be asserted, so test for + # existence in request log individually + self.assert_request(('PUT', self.cont_path)) + self.assert_request(('PUT', self.cont_path + '_segments', '', { + 'X-Auth-Token': 'bob_token', + 'X-Storage-Policy': 'foo', + 'Content-Length': '0', + })) + self.assert_request(('PUT', segment_path_0)) + self.assert_request(('PUT', segment_path_1)) + self.assert_request(('PUT', self.obj_path)) + self.assertTrue(self.obj[1:] in out.out) + expected_err = ("Warning: failed to create container '%s': 403 Fake\n" + "Warning: failed to create container '%s': 403 Fake" + ) % (self.cont, self.cont + '_segments') + self.assertEqual(expected_err, out.err.strip()) + def test_upload_with_no_access(self): fake_conn = self.fake_http_connection(403, 403) @@ -2143,6 +2515,38 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): self.assertTrue(expected_err in out.err) self.assertEqual('', out) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) + @mock.patch('swiftclient.service.Connection') + def test_download_bad_threads(self, mock_connection): + mock_connection.return_value.get_object.return_value = [{}, ''] + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'download', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'download', + cmd_args=[self.cont, self.obj, '--no-download'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + def test_download_with_read_write_access(self): req_handler = self._fake_cross_account_auth(True, True) fake_conn = self.fake_http_connection(403, on_request=req_handler, diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 5a6cbfa..cbb95db 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -18,7 +18,7 @@ import mock import six import socket import string -import testtools +import unittest import warnings import tempfile from hashlib import md5 @@ -34,7 +34,7 @@ import swiftclient.utils import swiftclient -class TestClientException(testtools.TestCase): +class TestClientException(unittest.TestCase): def test_is_exception(self): self.assertTrue(issubclass(c.ClientException, Exception)) @@ -51,6 +51,7 @@ class TestClientException(testtools.TestCase): 'status', 'reason', 'device', + 'response_content', ) for value in test_kwargs: kwargs = { @@ -59,6 +60,26 @@ class TestClientException(testtools.TestCase): exc = c.ClientException('test', **kwargs) self.assertIn(value, str(exc)) + def test_attrs(self): + test_kwargs = ( + 'scheme', + 'host', + 'port', + 'path', + 'query', + 'status', + 'reason', + 'device', + 'response_content', + 'response_headers', + ) + for value in test_kwargs: + key = 'http_%s' % value + kwargs = {key: value} + exc = c.ClientException('test', **kwargs) + self.assertIs(True, hasattr(exc, key)) + self.assertEqual(getattr(exc, key), value) + class MockHttpResponse(object): def __init__(self, status=0, headers=None, verify=False): @@ -251,12 +272,12 @@ class TestGetAuth(MockHttpTest): self.assertEqual(url, 'storageURL') self.assertEqual(token, 'someauthtoken') - e = self.assertRaises(c.ClientException, c.get_auth, - 'http://www.test.com/invalid_cert', - 'asdf', 'asdf', auth_version='1.0') + with self.assertRaises(c.ClientException) as exc_context: + c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', auth_version='1.0') # TODO: this test is really on validating the mock and not the # the full plumbing into the requests's 'verify' option - self.assertIn('invalid_certificate', str(e)) + self.assertIn('invalid_certificate', str(exc_context.exception)) def test_auth_v1_timeout(self): # this test has some overlap with @@ -491,6 +512,32 @@ class TestGetAuth(MockHttpTest): os_options=os_options, auth_version='2.0', insecure=False) + def test_auth_v2_cert(self): + os_options = {'tenant_name': 'foo'} + c.get_auth_keystone = fake_get_auth_keystone(os_options, None) + + auth_url_no_sslauth = 'https://www.tests.com' + auth_url_sslauth = 'https://www.tests.com/client-certificate' + + url, token = c.get_auth(auth_url_no_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie', cert_key='mickey') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie') + def test_auth_v3_with_tenant_name(self): # check the correct auth version is passed to get_auth_keystone os_options = {'tenant_name': 'asdf'} @@ -582,9 +629,12 @@ class TestHeadAccount(MockHttpTest): def test_server_error(self): body = 'c' * 65 - c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_account, - 'http://www.tests.com', 'asdf') + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_account('http://www.tests.com', 'asdf') + e = exc_context.exception self.assertEqual(e.http_response_content, body) self.assertEqual(e.http_status, 500) self.assertRequests([ @@ -617,17 +667,17 @@ class TestPostAccount(MockHttpTest): def test_server_error(self): body = 'c' * 65 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.post_account, - 'http://www.tests.com', 'asdf', {}) - self.assertEqual(e.http_response_content, body) - self.assertEqual(e.http_status, 500) + with self.assertRaises(c.ClientException) as exc_mgr: + c.post_account('http://www.tests.com', 'asdf', {}) + self.assertEqual(exc_mgr.exception.http_response_content, body) + self.assertEqual(exc_mgr.exception.http_status, 500) self.assertRequests([ ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) ]) # TODO: this is a fairly brittle test of the __repr__ on the # ClientException which should probably be in a targeted test new_body = "[first 60 chars of response] " + body[0:60] - self.assertEqual(e.__str__()[-89:], new_body) + self.assertEqual(exc_mgr.exception.__str__()[-89:], new_body) class TestGetContainer(MockHttpTest): @@ -740,14 +790,18 @@ class TestHeadContainer(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_container, - 'http://www.test.com', 'asdf', 'container') + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_container('http://www.test.com', 'asdf', 'container') + e = exc_context.exception self.assertRequests([ ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), ]) self.assertEqual(e.http_status, 500) self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) class TestPutContainer(MockHttpTest): @@ -764,10 +818,13 @@ class TestPutContainer(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.put_container, - 'http://www.test.com', 'token', 'container') - self.assertEqual(e.http_response_content, body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.put_container('http://www.test.com', 'token', 'container') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) self.assertRequests([ ('PUT', '/container', '', { 'x-auth-token': 'token', @@ -790,9 +847,14 @@ class TestDeleteContainer(MockHttpTest): class TestGetObject(MockHttpTest): def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.get_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_query_string(self): c.http_connection = self.fake_http_connection(200, @@ -870,6 +932,124 @@ class TestGetObject(MockHttpTest): self.assertRaises(StopIteration, next, resp) self.assertEqual(resp.read(), '') + def test_chunk_size_iter_chunked_no_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection( + 200, body='abcdef', headers={'Transfer-Encoding': 'chunked'}) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + # simulate a dropped connection + resp.resp.read() + self.assertRaises(StopIteration, next, resp) + + def test_chunk_size_iter_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection( + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}), + StubResponse(206, 'cdef', {'etag': 'some etag', + 'content-length': '4', + 'content-range': 'bytes 2-5/6'}), + StubResponse(206, 'ef', {'etag': 'some etag', + 'content-length': '2', + 'content-range': 'bytes 4-5/6'}), + ) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'cd') + self.assertEqual(2, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'ef') + self.assertEqual(3, conn.attempts) + self.assertRaises(StopIteration, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=4-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_chunk_size_iter_retry_no_range_support(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection(*[ + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}) + ] * 3) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'cd') + self.assertEqual(2, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'ef') + self.assertEqual(3, conn.attempts) + self.assertRaises(StopIteration, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=4-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_chunk_size_iter_retry_bad_range_response(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection( + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}), + StubResponse(206, 'abcdef', {'etag': 'some etag', + 'content-length': '6', + 'content-range': 'chunk 1-2/3'}) + ) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertRaises(c.ClientException, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + def test_get_object_with_resp_chunk_size_zero(self): def get_connection(self): def get_auth(): @@ -891,9 +1071,14 @@ class TestGetObject(MockHttpTest): class TestHeadObject(MockHttpTest): def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.head_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_request_headers(self): c.http_connection = self.fake_http_connection(204) @@ -935,7 +1120,7 @@ class TestPutObject(MockHttpTest): mock_file) text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, - 'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:fg:lp'} + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} resp = MockHttpResponse() conn[1].getresponse = resp.fake_response @@ -970,10 +1155,15 @@ class TestPutObject(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, c.put_object, *args) + with self.assertRaises(c.ClientException) as exc_context: + c.put_object(*args) + e = exc_context.exception self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { @@ -1151,13 +1341,16 @@ class TestPostObject(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) + delete_at = 2.1 # not str! we don't know what other devs will use! args = ('http://www.test.com', 'token', 'container', 'obj', - {'X-Object-Meta-Test': 'mymeta'}) + {'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}) c.post_object(*args) self.assertRequests([ ('POST', '/container/obj', '', { 'x-auth-token': 'token', - 'X-Object-Meta-Test': 'mymeta'}), + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}), ]) def test_unicode_ok(self): @@ -1169,7 +1362,7 @@ class TestPostObject(MockHttpTest): text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, b'X-Header2': 'value', - 'X-2': '1', 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr', + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', 'X-Object-Meta-Header-not-encoded': text, b'X-Object-Meta-Header-encoded': 'value'} @@ -1190,10 +1383,14 @@ class TestPostObject(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) args = ('http://www.test.com', 'token', 'container', 'obj', {}) - e = self.assertRaises(c.ClientException, c.post_object, *args) - self.assertEqual(e.http_response_content, body) + with self.assertRaises(c.ClientException) as exc_context: + c.post_object(*args) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) self.assertRequests([ ('POST', 'http://www.test.com/container/obj', '', { 'x-auth-token': 'token', @@ -1213,9 +1410,14 @@ class TestDeleteObject(MockHttpTest): ]) def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.delete_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_query_string(self): c.http_connection = self.fake_http_connection(200, @@ -1242,9 +1444,15 @@ class TestGetCapabilities(MockHttpTest): self.assertTrue(http_conn[1].resp.has_been_read) def test_server_error(self): - conn = self.fake_http_connection(500) + body = 'c' * 60 + headers = {'foo': 'bar'} + conn = self.fake_http_connection( + StubResponse(500, body, headers)) http_conn = conn('http://www.test.com/info') - self.assertRaises(c.ClientException, c.get_capabilities, http_conn) + with self.assertRaises(c.ClientException) as exc_context: + c.get_capabilities(http_conn) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_conn_get_capabilities_with_auth(self): auth_headers = { @@ -1347,17 +1555,23 @@ class TestHTTPConnection(MockHttpTest): def test_bad_url_scheme(self): url = u'www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "www.test.com"' self.assertEqual(expected, str(exc)) url = u'://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "://www.test.com"' self.assertEqual(expected, str(exc)) url = u'blah://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' self.assertEqual(expected, str(exc)) @@ -1389,6 +1603,15 @@ class TestHTTPConnection(MockHttpTest): conn = c.http_connection(u'http://www.test.com/', insecure=True) self.assertEqual(conn[1].requests_args['verify'], False) + def test_cert(self): + conn = c.http_connection(u'http://www.test.com/', cert='minnie') + self.assertEqual(conn[1].requests_args['cert'], 'minnie') + + def test_cert_key(self): + conn = c.http_connection( + u'http://www.test.com/', cert='minnie', cert_key='mickey') + self.assertEqual(conn[1].requests_args['cert'], ('minnie', 'mickey')) + def test_response_connection_released(self): _parsed_url, conn = c.http_connection(u'http://www.test.com/') conn.resp = MockHttpResponse() @@ -1524,8 +1747,9 @@ class TestConnection(MockHttpTest): } c.http_connection = self.fake_http_connection( *code_iter, headers=auth_resp_headers) - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, conn.retries + 1) # test default no-retry @@ -1533,8 +1757,9 @@ class TestConnection(MockHttpTest): 200, 498, headers=auth_resp_headers) conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) def test_resp_read_on_server_error(self): @@ -1804,9 +2029,10 @@ class TestConnection(MockHttpTest): # v2 auth timeouts = [] + os_options = {'tenant_name': 'tenant', 'auth_token': 'meta-token'} conn = c.Connection( 'http://auth.example.com', 'user', 'password', timeout=33.0, - os_options=dict(tenant_name='tenant'), auth_version=2.0) + os_options=os_options, auth_version=2.0) fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') with mock.patch('swiftclient.client._import_keystone_client', _make_fake_import_keystone_client(fake_ks)): @@ -1821,6 +2047,10 @@ class TestConnection(MockHttpTest): # check timeout passed to HEAD for account self.assertEqual(timeouts, [33.0]) + # check token passed to keystone client + self.assertIn('token', fake_ks.calls[0]) + self.assertEqual('meta-token', fake_ks.calls[0].get('token')) + def test_reset_stream(self): class LocalContents(object): @@ -1889,8 +2119,8 @@ class TestConnection(MockHttpTest): return '' def local_http_connection(url, proxy=None, cacert=None, - insecure=False, ssl_compression=True, - timeout=None): + insecure=False, cert=None, cert_key=None, + ssl_compression=True, timeout=None): parsed = urlparse(url) return parsed, LocalConnection() @@ -2127,9 +2357,86 @@ class TestLogging(MockHttpTest): def test_get_error(self): c.http_connection = self.fake_http_connection(404) - e = self.assertRaises(c.ClientException, c.get_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(e.http_status, 404) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 404) + + def test_redact_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_encoded = unicode_token_value.encode('utf8') + set_cookie_value = 'X-Auth-Token=%s' % token_value + set_cookie_encoded = set_cookie_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers={ + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded, + 'Etag': b'mock_etag', + 'Set-Cookie': set_cookie_encoded + } + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value[:16] + '...', output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value[:8] + '...', output) + self.assertIn('Set-Cookie', output) + self.assertIn(set_cookie_value[:16] + '...', output) + self.assertNotIn(token_value, output) + self.assertNotIn(unicode_token_value, output) + self.assertNotIn(set_cookie_value, output) + + def test_show_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + c.logger_settings['redact_sensitive_headers'] = False + unicode_token_encoded = unicode_token_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers=[ + ('X-Auth-Token', token_encoded), + ('X-Storage-Token', unicode_token_encoded), + ('Etag', b'mock_etag') + ] + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value, output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value, output) class TestCloseConnection(MockHttpTest): @@ -2325,6 +2632,28 @@ class TestServiceToken(MockHttpTest): actual['full_path']) self.assertEqual(conn.attempts, 1) + def test_service_token_get_container_full_listing(self): + # verify service token is sent with each request for a full listing + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, 200)): + with mock.patch('swiftclient.client.parse_api_response') as resp: + resp.side_effect = ([{"name": "obj1"}], []) + conn = self.get_connection() + conn.get_container('container1', full_listing=True) + self.assertEqual(2, len(self.request_log), self.request_log) + expected_urls = iter(( + 'http://storage_url.com/container1?format=json', + 'http://storage_url.com/container1?format=json&marker=obj1' + )) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual(next(expected_urls), + actual['full_path']) + self.assertEqual(conn.attempts, 1) + def test_service_token_head_container(self): with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index fe50f55..aae466c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import testtools +import unittest import mock import six import tempfile @@ -22,7 +22,7 @@ from hashlib import md5 from swiftclient import utils as u -class TestConfigTrueValue(testtools.TestCase): +class TestConfigTrueValue(unittest.TestCase): def test_TRUE_VALUES(self): for v in u.TRUE_VALUES: @@ -37,7 +37,7 @@ class TestConfigTrueValue(testtools.TestCase): self.assertIs(u.config_true_value(False), False) -class TestPrtBytes(testtools.TestCase): +class TestPrtBytes(unittest.TestCase): def test_zero_bytes(self): bytes_ = 0 @@ -119,7 +119,7 @@ class TestPrtBytes(testtools.TestCase): self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip()) -class TestTempURL(testtools.TestCase): +class TestTempURL(unittest.TestCase): def setUp(self): super(TestTempURL, self).setUp() @@ -164,7 +164,7 @@ class TestTempURL(testtools.TestCase): self.method) -class TestReadableToIterable(testtools.TestCase): +class TestReadableToIterable(unittest.TestCase): def test_iter(self): chunk_size = 4 @@ -216,7 +216,7 @@ class TestReadableToIterable(testtools.TestCase): self.assertEqual(actual_md5sum, data.get_md5sum()) -class TestLengthWrapper(testtools.TestCase): +class TestLengthWrapper(unittest.TestCase): def test_stringio(self): contents = six.StringIO(u'a' * 50 + u'b' * 50) @@ -292,7 +292,7 @@ class TestLengthWrapper(testtools.TestCase): self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) -class TestGroupers(testtools.TestCase): +class TestGroupers(unittest.TestCase): def test_n_at_a_time(self): result = list(u.n_at_a_time(range(100), 9)) self.assertEqual([9] * 11 + [1], list(map(len, result))) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 17e07ac..d04583f 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -18,7 +18,6 @@ from requests import RequestException from requests.structures import CaseInsensitiveDict from time import sleep import unittest -import testtools import mock import six from six.moves import reload_module @@ -58,6 +57,11 @@ def fake_get_auth_keystone(expected_os_options=None, exc=None, actual_kwargs['cacert'] is None: from swiftclient import client as c raise c.ClientException("unverified-certificate") + if auth_url.startswith("https") and \ + auth_url.endswith("client-certificate") and \ + not (actual_kwargs['cert'] and actual_kwargs['cert_key']): + from swiftclient import client as c + raise c.ClientException("noclient-certificate") return storage_url, token return fake_get_auth_keystone @@ -88,17 +92,19 @@ def fake_http_connect(*code_iter, **kwargs): def __init__(self, status, etag=None, body='', timestamp='1', headers=None): - self.status = status + self.status_code = self.status = status self.reason = 'Fake' + self.scheme = 'http' self.host = '1.2.3.4' self.port = '1234' self.sent = 0 self.received = 0 self.etag = etag - self.body = body + self.content = self.body = body self.timestamp = timestamp self._is_closed = True self.headers = headers or {} + self.request = None def getresponse(self): if kwargs.get('raise_exc'): @@ -189,7 +195,7 @@ def fake_http_connect(*code_iter, **kwargs): return connect -class MockHttpTest(testtools.TestCase): +class MockHttpTest(unittest.TestCase): def setUp(self): super(MockHttpTest, self).setUp() @@ -214,6 +220,7 @@ class MockHttpTest(testtools.TestCase): on_request = kwargs.get('on_request') def wrapper(url, proxy=None, cacert=None, insecure=False, + cert=None, cert_key=None, ssl_compression=True, timeout=None): if storage_url: self.assertEqual(storage_url, url) @@ -224,15 +231,18 @@ class MockHttpTest(testtools.TestCase): pass conn = RequestsWrapper() - def request(method, url, *args, **kwargs): + def request(method, path, *args, **kwargs): try: conn.resp = self.fake_connect() except StopIteration: self.fail('Unexpected %s request for %s' % ( - method, url)) - self.request_log.append((parsed, method, url, args, + method, path)) + self.request_log.append((parsed, method, path, args, kwargs, conn.resp)) conn.host = conn.resp.host + conn.resp.request = RequestsWrapper() + conn.resp.request.url = '%s://%s%s' % ( + conn.resp.scheme, conn.resp.host, path) conn.resp.has_been_read = False _orig_read = conn.resp.read @@ -241,15 +251,15 @@ class MockHttpTest(testtools.TestCase): return _orig_read(*args, **kwargs) conn.resp.read = read if on_request: - status = on_request(method, url, *args, **kwargs) + status = on_request(method, path, *args, **kwargs) conn.resp.status = status if auth_token: headers = args[1] self.assertEqual(auth_token, headers.get('X-Auth-Token')) if query_string: - self.assertTrue(url.endswith('?' + query_string)) - if url.endswith('invalid_cert') and not insecure: + self.assertTrue(path.endswith('?' + query_string)) + if path.endswith('invalid_cert') and not insecure: from swiftclient import client as c raise c.ClientException("invalid_certificate") if exc: @@ -499,8 +509,8 @@ class FakeKeystone(object): self.token = token class _Client(object): - def __init__(self, endpoint, token, **kwargs): - self.auth_token = token + def __init__(self, endpoint, auth_token, **kwargs): + self.auth_token = auth_token self.endpoint = endpoint self.service_catalog = self.ServiceCatalog(endpoint) @@ -515,8 +525,8 @@ class FakeKeystone(object): def Client(self, **kwargs): self.calls.append(kwargs) - self.client = self._Client(endpoint=self.endpoint, token=self.token, - **kwargs) + self.client = self._Client( + endpoint=self.endpoint, auth_token=self.token, **kwargs) return self.client class Unauthorized(Exception): @@ -1,18 +1,21 @@ [tox] -envlist = py27,py33,py34,py35,pypy,pep8 +envlist = py27,py34,py35,pypy,pep8 minversion = 1.6 skipsdist = True [testenv] usedevelop = True install_command = pip install -U {opts} {packages} -setenv = VIRTUAL_ENV={envdir} +setenv = + LANG=en_US.utf8 + VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = sh -c 'find . -not \( -type d -name .?\* -prune \) \ + .[keystone] +commands = sh -c '(find . -not \( -type d -name .?\* -prune \) \ \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ - -print0 | xargs -0 rm -rf' + -print0; find . -name "*.dbm*" -print0) | xargs -0 rm -rf' python setup.py testr --testr-args="{posargs}" whitelist_externals = sh passenv = SWIFT_* *_proxy |