summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/source/cli.rst361
-rw-r--r--doc/source/client-api.rst177
-rw-r--r--doc/source/index.rst24
-rw-r--r--doc/source/introduction.rst94
-rw-r--r--doc/source/sdk.rst48
-rw-r--r--doc/source/service-api.rst (renamed from doc/source/apis.rst)601
-rw-r--r--examples/capabilities.py20
-rw-r--r--examples/delete.py34
-rw-r--r--examples/download.py37
-rw-r--r--examples/list.py32
-rw-r--r--examples/post.py31
-rw-r--r--examples/stat.py25
-rw-r--r--examples/upload.py71
-rw-r--r--setup.cfg2
-rw-r--r--swiftclient/client.py71
-rw-r--r--swiftclient/multithreading.py1
-rw-r--r--swiftclient/service.py49
-rwxr-xr-xswiftclient/shell.py223
-rw-r--r--tests/unit/test_service.py204
-rw-r--r--tests/unit/test_shell.py179
-rw-r--r--tests/unit/test_swiftclient.py70
21 files changed, 1826 insertions, 528 deletions
diff --git a/doc/source/cli.rst b/doc/source/cli.rst
index 9527fbf..12de02f 100644
--- a/doc/source/cli.rst
+++ b/doc/source/cli.rst
@@ -1,29 +1,334 @@
-===
+====
CLI
-===
-
-Top-level commands
-~~~~~~~~~~~~~~~~~~
-
-.. TODO
-
- delete
- download
- list
- post
- stat
- upload
- info/capabilities
- tempurl
- auth
-
-Prescriptive examples
-~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO
-
- A "Hello World" example
- uploading an object
- creating a tempurl
- listing the contents of a container
- downloading an object \ No newline at end of file
+====
+
+The ``swift`` tool is a command line utility for communicating with an OpenStack
+Object Storage (swift) environment. It allows one to perform several types of
+operations.
+
+Authentication
+~~~~~~~~~~~~~~
+
+This section covers the options for authenticating with a swift
+object store. The combinations of options required for each authentication
+version are detailed below, but are just a subset of those that can be used
+to successfully authenticate. These are the most common and recommended
+combinations.
+
+You should obtain the details of your authentication version and credentials
+from your storage provider. These details should make it clearer which of the
+authentication sections below are most likely to allow you to connect to your
+storage account.
+
+Keystone v3
+-----------
+
+.. code-block:: bash
+
+ swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \
+ --os-project-name project1 --os-project-domain-name domain1 \
+ --os-username user --os-user-domain-name domain1 \
+ --os-password password list
+
+ swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \
+ --os-project-id 0123456789abcdef0123456789abcdef \
+ --os-user-id abcdef0123456789abcdef0123456789 \
+ --os-password password list
+
+Manually specifying the options above on the command line can be avoided by
+setting the following combinations of environment variables:
+
+.. code-block:: bash
+
+ ST_AUTH_VERSION=3
+ OS_USERNAME=user
+ OS_USER_DOMAIN_NAME=domain1
+ OS_PASSWORD=password
+ OS_PROJECT_NAME=project1
+ OS_PROJECT_DOMAIN_NAME=domain1
+ OS_AUTH_URL=https://api.example.com:5000/v3
+
+ ST_AUTH_VERSION=3
+ OS_USER_ID=abcdef0123456789abcdef0123456789
+ OS_PASSWORD=password
+ OS_PROJECT_ID=0123456789abcdef0123456789abcdef
+ OS_AUTH_URL=https://api.example.com:5000/v3
+
+Keystone v2
+-----------
+
+.. code-block:: bash
+
+ swift --os-auth-url https://api.example.com:5000/v2.0 \
+ --os-tenant-name tenant \
+ --os-username user --os-password password list
+
+Manually specifying the options above on the command line can be avoided by
+setting the following environment variables:
+
+.. code-block:: bash
+
+ ST_AUTH_VERSION=2.0
+ OS_USERNAME=user
+ OS_PASSWORD=password
+ OS_TENANT_NAME=tenant
+ OS_AUTH_URL=https://api.example.com:5000/v2.0
+
+Legacy auth systems
+-------------------
+
+You can configure swift to work with any number of other authentication systems
+that we will not cover in this document. If your storage provider is not using
+Keystone to provide access tokens, please contact them for instructions on the
+required options. It is likely that the options will need to be specified as
+below:
+
+.. code-block:: bash
+
+ swift -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key list
+
+Specifying the options above manually on the command line can be avoided by
+setting the following environment variables:
+
+.. code-block:: bash
+
+ ST_AUTH_VERSION=1.0
+ ST_AUTH=https://auth.api.rackspacecloud.com/v1.0
+ ST_USER=user
+ ST_KEY=key
+
+It is also possible that you need to use a completely separate auth system, in which
+case ``swiftclient`` cannot request a token for you. In this case you should make the
+authentication request separately and access your storage using the token and
+storage URL options shown below:
+
+.. code-block:: bash
+
+ swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \
+ --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \
+ list
+
+.. We need the backslash below in order to indent the note
+\
+
+ .. note::
+
+ Leftover environment variables are a common source of confusion when
+ authorization fails.
+
+CLI commands
+~~~~~~~~~~~~
+
+Stat
+----
+
+ ``stat [container [object]]``
+
+ Displays information for the account, container, or object depending on
+ the arguments given (if any). In verbose mode, the storage URL and the
+ authentication token are displayed as well.
+
+List
+----
+
+ ``list [command-options] [container]``
+
+ Lists the containers for the account or the objects for a container.
+ The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only
+ list items beginning with that prefix. The ``-d <delimiter>`` or
+ ``--delimiter <delimiter>`` is an option (for container listings only)
+ that will roll up items with the given delimiter (see `OpenStack Swift
+ general documentation <http://docs.openstack.org/developer/swift/>` for
+ what this means).
+
+ The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l``
+ and ``ls -lh``, the latter providing sizes in human readable format
+ (For example: ``3K``, ``12M``, etc). The latter two switches use more
+ overhead to retrieve the displayed details, which is directly proportional
+ to the number of container or objects listed.
+
+Upload
+------
+
+ ``upload [command-options] container file_or_directory [file_or_directory] [...]``
+
+ Uploads the files and directories specified by the remaining arguments to the
+ given container. The ``-c`` or ``--changed`` is an option that will only
+ upload files that have changed since the last upload. The
+ ``--object-name <object-name>`` is an option that will upload a file and
+ name object to ``<object-name>`` or upload a directory and use ``<object-name>``
+ as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and
+ ``--leave-segments`` are options as well (see ``--help`` for more).
+
+Post
+----
+
+ ``post [command-options] [container] [object]``
+
+ Updates meta information for the account, container, or object depending
+ on the arguments given. If the container is not found, the ``swiftclient``
+ will create it automatically, but this is not true for accounts and
+ objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl
+ <read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options.
+ The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects,
+ and is used to define the user metadata items to set in the form ``Name:Value``.
+ You can repeat this option. For example: ``post -m Color:Blue -m Size:Large``
+
+ For more information about ACL formats see the documentation:
+ `ACLs <http://docs.openstack.org/developer/swift/misc.html#acls/>`_.
+
+Download
+--------
+
+ ``download [command-options] [container] [object] [object] [...]``
+
+ Downloads everything in the account (with ``--all``), or everything in a
+ container, or a list of objects depending on the arguments given. For a
+ single object download, you may use the ``-o <filename>`` or ``--output <filename>``
+ option to redirect the output to a specific file or ``-`` to
+ redirect to stdout. You can specify optional headers with the repeatable
+ cURL-like option ``-H [--header <name:value>]``.
+
+Delete
+------
+
+ ``delete [command-options] [container] [object] [object] [...]``
+
+ Deletes everything in the account (with ``--all``), or everything in a
+ container, or a list of objects depending on the arguments given. Segments
+ of manifest objects will be deleted as well, unless you specify the
+ ``--leave-segments`` option.
+
+Capabilities
+------------
+
+ ``capabilities [proxy-url]``
+
+ Displays cluster capabilities. The output includes the list of the
+ activated Swift middlewares as well as relevant options for each ones.
+ Additionally the command displays relevant options for the Swift core. If
+ the ``proxy-url`` option is not provided, the storage URL retrieved after
+ authentication is used as ``proxy-url``.
+
+
+Examples
+~~~~~~~~
+
+In this section we present some example usage of the ``swift`` CLI. To keep the
+examples as short as possible, these examples assume that the relevant authentication
+options have been set using environment variables. You can obtain the full list of
+commands and options available in the ``swift`` CLI by executing the following:
+
+.. code-block:: bash
+
+ > swift --help
+ > swift <command> --help
+
+Simple examples
+---------------
+
+List the existing swift containers:
+
+.. code-block:: bash
+
+ > swift list
+
+ container_1
+
+Create a new container:
+
+.. code-block:: bash
+
+ > swift post TestContainer
+
+Upload an object into a container:
+
+.. code-block:: bash
+
+ > swift upload TestContainer testSwift.txt
+
+ testSwift.txt
+
+List the contents of a container:
+
+.. code-block:: bash
+
+ > swift list TestContainer
+
+ testSwift.txt
+
+Download an object from a container:
+
+.. code-block:: bash
+
+ > swift download TestContainer testSwift.txt
+
+ testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s]
+
+.. We need the backslash below in order to indent the note
+\
+
+ .. note::
+
+ To upload an object to a container, your current working directory must be
+ where the file is located or you must provide the complete path to the file.
+ In the case that you provide the complete path of the file, that complete
+ path will be the name of the uploaded object.
+
+For example:
+
+.. code-block:: bash
+
+ > swift upload TestContainer /home/swift/testSwift/testSwift.txt
+
+ home/swift/testSwift/testSwift.txt
+
+ > swift list TestContainer
+
+ home/swift/testSwift/testSwift.txt
+
+More complex examples
+---------------------
+
+Swift has a single object size limit of 5GiB. In order to upload files larger
+than this, we must create a large object that consists of smaller segments.
+The example below shows how to upload a large video file as a static large
+object in 1GiB segments:
+
+.. code-block:: bash
+
+ > swift upload videos --use-slo --segment-size 1G myvideo.mp4
+
+ myvideo.mp4 segment 8
+ myvideo.mp4 segment 4
+ myvideo.mp4 segment 2
+ myvideo.mp4 segment 7
+ myvideo.mp4 segment 0
+ myvideo.mp4 segment 1
+ myvideo.mp4 segment 3
+ myvideo.mp4 segment 6
+ myvideo.mp4 segment 5
+ myvideo.mp4
+
+This command will upload segments to a container named ``videos_segments``, and
+create a manifest file describing the entire object in the ``videos`` container.
+For more information on large objects, see the documentation `here
+<http://docs.openstack.org/developer/swift/overview_large_objects.html>`_.
+
+.. code-block:: bash
+
+ > swift list videos
+
+ myvideo.mp4
+
+ > swift list videos_segments
+
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007
+ myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008
diff --git a/doc/source/client-api.rst b/doc/source/client-api.rst
new file mode 100644
index 0000000..5677f70
--- /dev/null
+++ b/doc/source/client-api.rst
@@ -0,0 +1,177 @@
+==============================
+The swiftclient.Connection API
+==============================
+
+A low level API that provides methods for authentication and methods that
+correspond to the individual REST API calls described in the swift
+documentation.
+
+For usage details see the client docs: :mod:`swiftclient.client`.
+
+Authentication
+--------------
+
+This section covers the various combinations of kwargs required when creating
+and instance of the ``Connection`` object for communicating with a swift
+object store. The combinations of options required for each authentication
+version are detailed below, but are
+just a subset of those that can be used to successfully authenticate. These
+are the most common and recommended combinations.
+
+Keystone v3
+~~~~~~~~~~~
+
+.. code-block:: python
+
+ _authurl = 'http://127.0.0.1:5000/v3/'
+ _auth_version = '3'
+ _user = 'tester'
+ _key = 'testing'
+ _os_options = {
+ 'user_domain_name': 'Default',
+ 'project_domain_name': 'Default',
+ 'project_name': 'Default'
+ }
+
+ conn = Connection(
+ authurl=_authurl,
+ user=_user,
+ key=_key,
+ os_options=_os_options,
+ auth_version=_auth_version
+ )
+
+.. code-block:: python
+
+ _authurl = 'http://127.0.0.1:5000/v3/'
+ _auth_version = '3'
+ _user = 'tester'
+ _key = 'testing'
+ _os_options = {
+ 'user_domain_id': 'Default',
+ 'project_domain_id': 'Default',
+ 'project_id': 'Default'
+ }
+
+ conn = Connection(
+ authurl=_authurl,
+ user=_user,
+ key=_key,
+ os_options=_os_options,
+ auth_version=_auth_version
+ )
+
+Keystone v2
+~~~~~~~~~~~
+
+.. code-block:: python
+
+ _authurl = 'http://127.0.0.1:5000/v2.0/'
+ _auth_version = '2'
+ _user = 'tester'
+ _key = 'testing'
+ _tenant_name = 'test'
+
+ conn = Connection(
+ authurl=_authurl,
+ user=_user,
+ key=_key,
+ tenant_name=_tenant_name,
+ auth_version=_auth_version
+ )
+
+Legacy Auth
+~~~~~~~~~~~
+
+.. code-block:: python
+
+ _authurl = 'http://127.0.0.1:8080/'
+ _auth_version = '1'
+ _user = 'tester'
+ _key = 'testing'
+ _tenant_name = 'test'
+
+ conn = Connection(
+ authurl=_authurl,
+ user=_user,
+ key=_key,
+ tenant_name=_tenant_name,
+ auth_version=_auth_version
+ )
+
+Examples
+--------
+
+In this section we present some simple code examples that demonstrate the usage
+of the ``Connection`` API. You can find full details of the options and methods
+available to the ``Connection`` API in the docstring generated documentation:
+:mod:`swiftclient.client`.
+
+List the available containers:
+
+.. code-block:: python
+
+ resp_headers, containers = conn.get_account()
+ print("Response headers: %s" % resp_headers)
+ for container in containers:
+ print(container)
+
+Create a new container:
+
+.. code-block:: python
+
+ container = 'new-container'
+ conn.put_container(container)
+ resp_headers, containers = conn.get_account()
+ if container in containers:
+ print("The container was created")
+
+Create a new object with the contents of a local text file:
+
+.. code-block:: python
+
+ container = 'new-container'
+ with open('local.txt', 'r') as local:
+ conn.put_object(
+ container,
+ 'local_object.txt',
+ contents=local,
+ content_type='text/plain'
+ )
+
+Confirm presence of the object:
+
+.. code-block:: python
+
+ obj = 'local_object.txt'
+ container = 'new-container'
+ try:
+ resp_headers = conn.head_object(container, obj)
+ print('The object was successfully created')
+ except ClientException as e:
+ if e.http_status = '404':
+ print('The object was not found')
+ else:
+ print('An error occurred checking for the existence of the object')
+
+Download the created object:
+
+.. code-block:: python
+
+ obj = 'local_object.txt'
+ container = 'new-container'
+ resp_headers, obj_contents = conn.get_object(container, obj)
+ with open('local_copy.txt', 'w') as local:
+ local.write(obj_contents)
+
+Delete the created object:
+
+.. code-block:: python
+
+ obj = 'local_object.txt'
+ container = 'new-container'
+ try:
+ conn.delete_object(container, obj)
+ print("Successfully deleted the object")
+ except ClientException as e:
+ print("Failed to delete the object with error: %s" % e)
diff --git a/doc/source/index.rst b/doc/source/index.rst
index da16a3c..f123b7b 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -1,18 +1,28 @@
+======================================
Welcome to the python-swiftclient Docs
-**************************************
+======================================
+
+Introduction
+~~~~~~~~~~~~
+
+.. toctree::
+ :maxdepth: 2
+
+ introduction
Developer Documentation
-=======================
+~~~~~~~~~~~~~~~~~~~~~~~
.. toctree::
:maxdepth: 2
- apis
cli
- sdk
+ service-api
+ client-api
+
Code-Generated Documentation
-============================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. toctree::
:maxdepth: 2
@@ -20,14 +30,14 @@ Code-Generated Documentation
swiftclient
Indices and tables
-==================
+~~~~~~~~~~~~~~~~~~
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
License
-=======
+~~~~~~~
Copyright 2013 OpenStack, LLC.
diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst
new file mode 100644
index 0000000..926b1b9
--- /dev/null
+++ b/doc/source/introduction.rst
@@ -0,0 +1,94 @@
+============
+Introduction
+============
+
+Where to Start?
+~~~~~~~~~~~~~~~
+
+The ``python-swiftclient`` project comprises a command line tool and two
+separate APIs for accessing swift programmatically. Choosing the most
+appropriate method for a given use case is the first problem a user needs to
+solve.
+
+Use Cases
+---------
+
+Alongside the command line tool, the ``python-swiftclient`` includes two
+levels of API:
+
+ * A low level client API that provides simple Python wrappers around the
+ various authentication mechanisms and the individual HTTP requests.
+ * A high level service API that provides methods for performing common
+ operations in parallel on a thread pool.
+
+Example use cases:
+
+ * Uploading and retrieving data
+ Use the command line tool if you are simply uploading and downloading
+ files and directories to and from your filesystem. The command line tool
+ can be integrated into a shell script to automate tasks.
+
+ * Integrating into an automated Python workflow
+ Use the ``SwiftService`` API to perform operations offered by the CLI
+ if your use case requires integration with a Python-based workflow.
+ This method offers greater control and flexibility over individual object
+ operations, such as the metadata set on each object. The ``SwiftService``
+ class provides methods to perform multiple sets of operations against a
+ swift object store using a configurable shared thread pool. A single
+ instance of the ``SwiftService`` class can be shared between multiple
+ threads in your own code.
+
+ * Developing an application in Python to access a swift object store
+ Use the ``SwiftService`` API to develop Python applications that use
+ swift to store and retrieve objects. A ``SwiftService`` instance provides
+ a configurable thread pool for performing all operations supported by the
+ CLI.
+
+ * Fine-grained control over threading or the requests being performed
+ Use the ``Connection`` API if your use case requires fine grained control
+ over advanced features or you wish to use your own existing threading
+ model. Examples of advanced features requiring the use of the
+ ``Connection`` API include creating an SLO manifest that references
+ already existing objects, or fine grained control over the query strings
+ supplied with each HTTP request.
+
+Important considerations
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This section covers some important considerations, helpful hints, and things to
+avoid when integrating an object store into your workflow.
+
+An object store is not a filesystem
+-----------------------------------
+
+It cannot be stressed enough that your usage of the object store should reflect
+the proper use case, and not treat the storage like a traditional filesystem.
+There are two main restrictions to bear in mind when designing an application
+that uses an object store:
+
+ * You cannot rename objects. Due to fact that the name of an object is one
+ of the factors that determines where the object and its replicas are stored,
+ renaming would require multiple copies of the data to be moved between
+ physical storage devices. If you want to rename an object you must upload
+ to the new location, or make a server side copy request to the new location,
+ and then delete the original.
+
+ * You cannot modify objects. Objects are stored in multiple locations and
+ are checked for integrity based on the MD5 sum calculated during
+ upload. In order to modify the contents of an object, the entire desired
+ contents must be re-uploaded. In certain special cases it is possible to
+ work around this restriction using large objects, but no general
+ file-like access is available to modify a stored object.
+
+Objects cannot be locked
+------------------------
+
+There is no mechanism to perform a combination of reading the
+data/metadata from an object and writing an update to that data/metadata in an
+atomic way. Any user with access to a container could update the contents or
+metadata associated with an object at any time.
+
+Workflows that assume that no updates have been made since the last read of an
+object should be discouraged. Enabling a workflow of this type requires an
+external object locking mechanism and/or cooperation between all clients
+accessing the data.
diff --git a/doc/source/sdk.rst b/doc/source/sdk.rst
deleted file mode 100644
index aa15250..0000000
--- a/doc/source/sdk.rst
+++ /dev/null
@@ -1,48 +0,0 @@
-===
-SDK
-===
-
-Where to start?
-~~~~~~~~~~~~~~~
-
-.. TODO
-
- when to use SwiftService
- when to use client.py
-
-SwiftService classes and methods
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO
-
- docs for each method (autogen from docstrings?)
-
-Client classes and methods
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO
-
- docs for each method (autogen from docstrings?)
-
-Guidelines for writing an app
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO
-
- auth
- how to use various features
- when to use various features
- pooling connections
- concurrency
- retries
-
-Prescriptive examples
-~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO
-
- A "Hello World" example
- connecting
- uploading an object
- uploading a directory
- \ No newline at end of file
diff --git a/doc/source/apis.rst b/doc/source/service-api.rst
index 935b4a4..7d65fd1 100644
--- a/doc/source/apis.rst
+++ b/doc/source/service-api.rst
@@ -1,63 +1,93 @@
-======================
-python-swiftclient API
-======================
+================================
+The swiftclient.SwiftService API
+================================
-The python-swiftclient includes two levels of API. A low level client API that
-provides simple python wrappers around the various authentication mechanisms,
-the individual HTTP requests, and a high level service API that provides
-methods for performing common operations in parallel on a thread pool.
+A higher-level API aimed at allowing developers an easy way to perform multiple
+operations asynchronously using a configurable thread pool. Documentation for
+each service method call can be found here: :mod:`swiftclient.service`.
-This document aims to provide guidance for choosing between these APIs and
-examples of usage for the service API.
+Authentication
+--------------
+This section covers the various options for authenticating with a swift
+object store. The combinations of options required for each authentication
+version are detailed below. Once again, these are just a subset of those that
+can be used to successfully authenticate, but they are the most common and
+recommended.
-Important Considerations
-~~~~~~~~~~~~~~~~~~~~~~~~
+The relevant authentication options are presented as python dictionaries that
+should be added to any other options you are supplying to your ``SwiftService``
+instance. As indicated in the python code, you can also set these options as
+environment variables that will be loaded automatically if the relevant option
+is not specified.
-This section covers some important considerations, helpful hints, and things
-to avoid when integrating an object store into your workflow.
+The ``SwiftService`` authentication attempts to automatically select
+the auth version based on the combination of options specified, but
+supplying options from multiple different auth versions can cause unexpected
+behaviour.
-An Object Store is not a filesystem
------------------------------------
+ .. note::
-.. important::
+ Leftover environment variables are a common source of confusion when
+ authorization fails.
- It cannot be stressed enough that your usage of the object store should reflect
- the use case, and not treat the storage like a filesystem.
+Keystone V3
+~~~~~~~~~~~
-There are 2 main restrictions to bear in mind here when designing your use of the object
-store:
+.. code-block:: python
-#. Objects cannot be renamed due to the way in which objects are stored and
- references by the object store. This usually requires multiple copies of
- the data to be moved between physical storage devices.
- As a result, a move operation is not provided. If the user wants to move an
- object they must re-upload to the new location and delete the
- original.
-#. Objects cannot be modified. Objects are stored in multiple locations and are
- checked for integrity based on the ``MD5 sum`` calculated during upload.
- Object creation is a 1-shot event, and in order to modify the contents of an
- object the entire new contents must be re-uploaded. In certain special cases
- it is possible to work around this restriction using large objects, but no
- general file-like access is available to modify a stored object.
+ {
+ ...
+ "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '3'
+ "os_username": environ.get('OS_USERNAME'),
+ "os_password": environ.get('OS_PASSWORD'),
+ "os_project_name": environ.get('OS_PROJECT_NAME'),
+ "os_project_domain_name": environ.get('OS_PROJECT_DOMAIN_NAME'),
+ "os_auth_url": environ.get('OS_AUTH_URL'),
+ ...
+ }
+.. code-block:: python
-The swiftclient.Connection API
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ {
+ ...
+ "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '3'
+ "os_username": environ.get('OS_USERNAME'),
+ "os_password": environ.get('OS_PASSWORD'),
+ "os_project_id": environ.get('OS_PROJECT_ID'),
+ "os_project_domain_id": environ.get('OS_PROJECT_DOMAIN_ID'),
+ "os_auth_url": environ.get('OS_AUTH_URL'),
+ ...
+ }
-A low level API that provides methods for authentication and methods that
-correspond to the individual REST API calls described in the swift
-documentation.
+Keystone V2
+~~~~~~~~~~~
-For usage details see the client docs: :mod:`swiftclient.client`.
+.. code-block:: python
+ {
+ ...
+ "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '2.0'
+ "os_username": environ.get('OS_USERNAME'),
+ "os_password": environ.get('OS_PASSWORD'),
+ "os_tenant_name": environ.get('OS_TENANT_NAME'),
+ "os_auth_url": environ.get('OS_AUTH_URL'),
+ ...
+ }
-The swiftclient.SwiftService API
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Legacy Auth
+~~~~~~~~~~~
-A higher level API aimed at allowing developers an easy way to perform multiple
-operations asynchronously using a configurable thread pool. Documentation for each
-service method call can be found here: :mod:`swiftclient.service`.
+.. code-block:: python
+
+ {
+ ...
+ "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '1.0'
+ "auth": environ.get('ST_AUTH'),
+ "user": environ.get('ST_USER'),
+ "key": environ.get('ST_KEY'),
+ ...
+ }
Configuration
-------------
@@ -77,7 +107,7 @@ passed to the ``SwiftService`` during initialisation. The options available
in this dictionary are described below, along with their defaults:
Options
-^^^^^^^
+~~~~~~~
``retries``: ``5``
The number of times that the library should attempt to retry HTTP
@@ -192,51 +222,8 @@ source code for ``python-swiftclient``. Each ``SwiftService`` method also allows
for an optional dictionary to override those specified at init time, and the
appropriate docstrings show which options modify each method's behaviour.
-Authentication
-~~~~~~~~~~~~~~
-
-This section covers the various options for authenticating with a swift
-object store. The combinations of options required for each authentication
-version are detailed below.
-
-Version 1.0 Auth
-----------------
-
- ``auth_version``: ``environ.get('ST_AUTH_VERSION')``
-
- ``auth``: ``environ.get('ST_AUTH')``
-
- ``user``: ``environ.get('ST_USER')``
-
- ``key``: ``environ.get('ST_KEY')``
-
-
-Version 2.0 and 3.0 Auth
-------------------------
-
- ``auth_version``: ``environ.get('ST_AUTH_VERSION')``
-
- ``os_username``: ``environ.get('OS_USERNAME')``
-
- ``os_password``: ``environ.get('OS_PASSWORD')``
-
- ``os_tenant_name``: ``environ.get('OS_TENANT_NAME')``
-
- ``os_auth_url``: ``environ.get('OS_AUTH_URL')``
-
-As is evident from the default values, if these options are not set explicitly
-in the options dictionary, then they will default to the values of the given
-environment variables. The ``SwiftService`` authentication automatically selects
-the auth version based on the combination of options specified, but
-having options from different auth versions can cause unexpected behaviour.
-
- .. note::
-
- Leftover environment variables are a common source of confusion when
- authorization fails.
-
-Operation Return Values
-~~~~~~~~~~~~~~~~~~~~~~~
+Available Operations
+--------------------
Each operation provided by the service API may raise a ``SwiftError`` or
``ClientException`` for any call that fails completely (or a call which
@@ -371,32 +358,14 @@ operation was not successful, and will include the keys below:
}
Example
--------
+^^^^^^^
The code below demonstrates the use of ``stat`` to retrieve the headers for a
given list of objects in a container using 20 threads. The code creates a
-mapping from object name to headers.
-
-.. code-block:: python
+mapping from object name to headers which is then pretty printed to the log.
- import logging
-
- from swiftclient.service import SwiftService
-
- logger = logging.getLogger()
- _opts = {'object_dd_threads': 20}
- with SwiftService(options=_opts) as swift:
- container = 'container1'
- objects = [ 'object_%s' % n for n in range(0,100) ]
- header_data = {}
- stats_it = swift.stat(container=container, objects=objects)
- for stat_res in stats_it:
- if stat_res['success']:
- header_data[stat_res['object']] = stat_res['headers']
- else:
- logger.error(
- 'Failed to retrieve stats for %s' % stat_res['object']
- )
+.. literalinclude:: ../../examples/stat.py
+ :language: python
List
~~~~
@@ -456,55 +425,38 @@ dictionary as described below:
}
Example
--------
+^^^^^^^
The code below demonstrates the use of ``list`` to list all items in a
container that are over 10MiB in size:
-.. code-block:: python
-
- container = 'example_container'
- minimum_size = 10*1024**2
- with SwiftService() as swift:
- try:
- stats_parts_gen = swift.list(container=container)
- for stats in stats_parts_gen:
- if stats["success"]:
- for item in stats["listing"]:
- i_size = int(item["bytes"])
- if i_size > minimum_size:
- i_name = item["name"]
- i_etag = item["hash"]
- print(
- "%s [size: %s] [etag: %s]" %
- (i_name, i_size, i_etag)
- )
- else:
- raise stats["error"]
- except SwiftError as e:
- output_manager.error(e.value)
+.. literalinclude:: ../../examples/list.py
+ :language: python
Post
~~~~
Post can be called against an account, container or list of objects in order to
-update the metadata attached to the given items. Each element of the object list
-may be a plain string of the object name, or a ``SwiftPostObject`` that
-allows finer control over the options applied to each of the individual post
-operations. In the first two cases a single dictionary is returned containing the
-results of the operation, and in the case of a list of objects being supplied,
-an iterator over the results generated for each object post is returned. If the
-given container or account does not exist, the ``post`` method will raise a
-``SwiftError``.
-
-.. When a string is given for the object name, the options
-
-Successful metadata update results are dictionaries as described below:
+update the metadata attached to the given items. In the first two cases a single
+dictionary is returned containing the results of the operation, and in the case
+of a list of objects being supplied, an iterator over the results generated for
+each object post is returned.
+
+Each element of the object list may be a plain string of the object name, or a
+``SwiftPostObject`` that allows finer control over the options and metadata
+applied to each of the individual post operations. When a string is given for
+the object name, the options and metadata applied are a combination of those
+supplied to the call to ``post()`` and the defaults of the ``SwiftService``
+object.
+
+If the given container or account does not exist, the ``post`` method will
+raise a ``SwiftError``. Successful metadata update results are dictionaries as
+described below:
.. code-block:: python
{
- 'action': <'post_account'|<'post_container'>|'post_object'>,
+ 'action': <'post_account'|'post_container'|'post_object'>,
'success': True,
'container': <container>,
'object': <object>,
@@ -513,28 +465,87 @@ Successful metadata update results are dictionaries as described below:
}
.. note::
-
Updating user metadata keys will not only add any specified keys, but
will also remove user metadata that has previously been set. This means
that each time user metadata is updated, the complete set of desired
key-value pairs must be specified.
+Example
+^^^^^^^
+The code below demonstrates the use of ``post`` to set an archive folder in a
+given container to expire after a 24 hour delay:
-.. Example
-.. -------
+.. literalinclude:: ../../examples/post.py
+ :language: python
-.. TBD
+Download
+~~~~~~~~
+
+Download can be called against an entire account, a single container, or a list
+of objects in a given container. Each element of the object list is a string
+detailing the full name of an object to download.
+
+In order to download the full contents of an entire account, you must set the
+value of ``yes_all`` to ``True`` in the ``options`` dictionary supplied to
+either the ``SwiftService`` instance or the call to ``download``.
+
+If the given container or account does not exist, the ``download`` method will
+raise a ``SwiftError``, otherwise an iterator over the results generated for
+each object download is returned.
+
+See :mod:`swiftclient.service.SwiftService.download` for docs generated from the
+method docstring.
+
+For each successfully downloaded object, the results returned by the iterator
+will be a dictionary as described below (results are not returned for completed
+container or object segment downloads):
+
+.. code-block:: python
+
+ {
+ 'action': 'download_object',
+ 'container': <container>,
+ 'object': <object name>,
+ 'success': True,
+ 'path': <local path to downloaded object>,
+ 'pseudodir': <if true, the download created an empty directory>,
+ 'start_time': <time download started>,
+ 'end_time': <time download completed>,
+ 'headers_receipt': <time the headers from the object were retrieved>,
+ 'auth_end_time': <time authentication completed>,
+ 'read_length': <bytes_read>,
+ 'attempts': <attempt count>,
+ 'response_dict': <HTTP response details>
+ }
-.. Download
-.. ~~~~~~~~
+Any failure uploading an object will return a failure dictionary as described
+below:
-.. TBD
+.. code-block:: python
-.. Example
-.. -------
+ {
+ 'action': 'download_object',
+ 'container': <container>,
+ 'object': <object name>,
+ 'success': False,
+ 'path': <local path of the failed download>,
+ 'pseudodir': <if true, the failed download was an empty directory>,
+ 'attempts': <attempt count>,
+ 'error': <error>,
+ 'traceback': <trace>,
+ 'error_timestamp': <timestamp>,
+ 'response_dict': <HTTP response details>
+ }
-.. TBD
+Example
+^^^^^^^
+
+The code below demonstrates the use of ``download`` to download all PNG images
+from a dated archive folder in a given container:
+
+.. literalinclude:: ../../examples/download.py
+ :language: python
Upload
~~~~~~
@@ -550,7 +561,7 @@ the upload are those supplied to the call to ``upload``.
Constructing a ``SwiftUploadObject`` allows the user to supply an object name
for the uploaded file, and modify the options used by ``upload`` at the
-granularity of invidivual files.
+granularity of individual files.
If the given container or account does not exist, the ``upload`` method will
raise a ``SwiftError``, otherwise an iterator over the results generated for
@@ -622,97 +633,195 @@ below:
}
Example
--------
+^^^^^^^
The code below demonstrates the use of ``upload`` to upload all files and
-folders in ``/tmp``, and renaming each object by replacing ``/tmp`` in the
-object or directory marker names with ``temporary-objects``:
+folders in a given directory, and rename each object by replacing the root
+directory name with 'my-<d>-objects', where <d> is the name of the uploaded
+directory:
+
+.. literalinclude:: ../../examples/upload.py
+ :language: python
+
+Delete
+~~~~~~
+
+Delete can be called against an account or a container to remove the containers
+or objects contained within them. Each call to ``delete`` returns an iterator
+over results of each resulting sub-request.
+
+If the number of requested delete operations is large and the target swift
+cluster is running the bulk middleware, the call to ``SwiftService.delete`` will
+make use of bulk operations and the returned result iterator will return
+``bulk_delete`` results rather than individual ``delete_object``,
+``delete_container`` or ``delete_segment`` results.
+
+See :mod:`swiftclient.service.SwiftService.delete` for docs generated from the
+method docstring.
+
+For each successfully deleted container, object or segment, the results returned
+by the iterator will be a dictionary as described below:
+
+.. code-block:: python
+
+ {
+ 'action': <'delete_object'|'delete_segment'>,
+ 'container': <container>,
+ 'object': <object name>,
+ 'success': True,
+ 'attempts': <attempt count>,
+ 'response_dict': <HTTP response details>
+ }
+
+ {
+ 'action': 'delete_container',
+ 'container': <container>,
+ 'success': True,
+ 'response_dict': <HTTP response details>,
+ 'attempts': <attempt count>
+ }
+
+ {
+ 'action': 'bulk_delete',
+ 'container': <container>,
+ 'objects': <[objects]>,
+ 'success': True,
+ 'attempts': <attempt count>,
+ 'response_dict': <HTTP response details>
+ }
+
+Any failure in a delete operation will return a failure dictionary as described
+below:
.. code-block:: python
- _opts['object_uu_threads'] = 20
- with SwiftService(options=_opts) as swift, OutputManager() as out_manager:
- try:
- # Collect all the files and folders in '/tmp'
- objs = []
- dir_markers = []
- dir = '/tmp':
- for (_dir, _ds, _fs) in walk(f):
- if not (_ds + _fs):
- dir_markers.append(_dir)
- else:
- objs.extend([join(_dir, _f) for _f in _fs])
-
- # Now that we've collected all the required files and dir markers
- # build the ``SwiftUploadObject``s for the call to upload
- objs = [
- SwiftUploadObject(
- o, object_name=o.replace(
- '/tmp', 'temporary-objects', 1
- )
- ) for o in objs
- ]
- dir_markers = [
- SwiftUploadObject(
- None, object_name=d.replace(
- '/tmp', 'temporary-objects', 1
- ), options={'dir_marker': True}
- ) for d in dir_markers
- ]
-
- # Schedule uploads on the SwiftService thread pool and iterate
- # over the results
- for r in swift.upload(container, objs + dir_markers):
- if r['success']:
- if 'object' in r:
- out_manager.print_msg(r['object'])
- elif 'for_object' in r:
- out_manager.print_msg(
- '%s segment %s' % (r['for_object'],
- r['segment_index'])
- )
- else:
- error = r['error']
- if r['action'] == "create_container":
- out_manager.warning(
- 'Warning: failed to create container '
- "'%s'%s", container, msg
- )
- elif r['action'] == "upload_object":
- out_manager.error(
- "Failed to upload object %s to container %s: %s" %
- (container, r['object'], error)
- )
- else:
- out_manager.error("%s" % error)
-
- except SwiftError as e:
- out_manager.error(e.value)
-
-.. Delete
-.. ~~~~~~
-.. Do we want to hide this section until it is complete?
-
-.. TBD
-
-.. Example
-.. -------
-
-.. Do we want to hide this section until it is complete?
-
-.. TBD
-
-.. Capabilities
-.. ~~~~~~~~~~~~
-
-.. Do we want to hide this section until it is complete?
-
-.. TBD
-
-.. Example
-.. -------
-
-.. Do we want to hide this section until it is complete?
-
-.. TBD
+ {
+ 'action': ('delete_object'|'delete_segment'),
+ 'container': <container>,
+ 'object': <object name>,
+ 'success': False,
+ 'attempts': <attempt count>,
+ 'error': <error>,
+ 'traceback': <trace>,
+ 'error_timestamp': <timestamp>,
+ 'response_dict': <HTTP response details>
+ }
+
+ {
+ 'action': 'delete_container',
+ 'container': <container>,
+ 'success': False,
+ 'error': <error>,
+ 'traceback': <trace>,
+ 'error_timestamp': <timestamp>,
+ 'response_dict': <HTTP response details>,
+ 'attempts': <attempt count>
+ }
+
+ {
+ 'action': 'bulk_delete',
+ 'container': <container>,
+ 'objects': <[objects]>,
+ 'success': False,
+ 'attempts': <attempt count>,
+ 'error': <error>,
+ 'traceback': <trace>,
+ 'error_timestamp': <timestamp>,
+ 'response_dict': <HTTP response details>
+ }
+
+Example
+^^^^^^^
+
+The code below demonstrates the use of ``delete`` to remove a given list of
+objects from a specified container. As the objects are deleted the transaction
+id of the relevant request is printed along with the object name and number
+of attempts required. By printing the transaction id, the printed operations
+can be easily linked to events in the swift server logs:
+
+.. literalinclude:: ../../examples/delete.py
+ :language: python
+
+Capabilities
+~~~~~~~~~~~~
+
+Capabilities can be called against an account or a particular proxy URL in
+order to determine the capabilities of the swift cluster. These capabilities
+include details about configuration options and the middlewares that are
+installed in the proxy pipeline.
+
+See :mod:`swiftclient.service.SwiftService.capabilities` for docs generated from
+the method docstring.
+
+For each successful call to list capabilities, a result dictionary will be
+returned with the contents described below:
+
+ {
+ 'action': 'capabilities',
+ 'timestamp': <time of the call>,
+ 'success': True,
+ 'capabilities': <dictionary containing capability details>
+ }
+
+The contents of the capabilities dictionary contain the core swift capabilities
+under the key ``swift``, all other keys show the configuration options for
+additional middlewares deployed in the proxy pipeline. An example capabilities
+dictionary is given below:
+
+.. code-block:: python
+
+ {
+ 'account_quotas': {},
+ 'bulk_delete': {
+ 'max_deletes_per_request': 10000,
+ 'max_failed_deletes': 1000
+ },
+ 'bulk_upload': {
+ 'max_containers_per_extraction': 10000,
+ 'max_failed_extractions': 1000
+ },
+ 'container_quotas': {},
+ 'container_sync': {'realms': {}},
+ 'formpost': {},
+ 'keystoneauth': {},
+ 'slo': {
+ 'max_manifest_segments': 1000,
+ 'max_manifest_size': 2097152,
+ 'min_segment_size': 1048576
+ },
+ 'swift': {
+ 'account_autocreate': True,
+ 'account_listing_limit': 10000,
+ 'allow_account_management': True,
+ 'container_listing_limit': 10000,
+ 'extra_header_count': 0,
+ 'max_account_name_length': 256,
+ 'max_container_name_length': 256,
+ 'max_file_size': 5368709122,
+ 'max_header_size': 8192,
+ 'max_meta_count': 90,
+ 'max_meta_name_length': 128,
+ 'max_meta_overall_size': 4096,
+ 'max_meta_value_length': 256,
+ 'max_object_name_length': 1024,
+ 'policies': [
+ {'default': True, 'name': 'Policy-0'}
+ ],
+ 'strict_cors_mode': False,
+ 'version': '2.2.2'
+ },
+ 'tempurl': {
+ 'methods': ['GET', 'HEAD', 'PUT']
+ }
+ }
+
+Example
+^^^^^^^
+
+The code below demonstrates the us of ``capabilities`` to determine if the
+Swift cluster supports static large objects, and if so, the maximum number of
+segments that can be described in a single manifest file, along with the
+size restrictions on those objects:
+.. literalinclude:: ../../examples/capabilities.py
+ :language: python
diff --git a/examples/capabilities.py b/examples/capabilities.py
new file mode 100644
index 0000000..024ad6f
--- /dev/null
+++ b/examples/capabilities.py
@@ -0,0 +1,20 @@
+import logging
+
+from swiftclient.exceptions import ClientException
+from swiftclient.service import SwiftService
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+with SwiftService() as swift:
+ try:
+ capabilities_result = swift.capabilities()
+ capabilities = capabilities_result['capabilities']
+ if 'slo' in capabilities:
+ print('SLO is supported')
+ else:
+ print('SLO is not supported')
+ except ClientException as e:
+ logger.error(e.value)
diff --git a/examples/delete.py b/examples/delete.py
new file mode 100644
index 0000000..6979d9e
--- /dev/null
+++ b/examples/delete.py
@@ -0,0 +1,34 @@
+import logging
+
+from swiftclient.service import SwiftService
+from sys import argv
+
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+_opts = {'object_dd_threads': 20}
+container = argv[1]
+objects = argv[2:]
+with SwiftService(options=_opts) as swift:
+ del_iter = swift.delete(container=container, objects=objects)
+ for del_res in del_iter:
+ c = del_res.get('container', '')
+ o = del_res.get('object', '')
+ a = del_res.get('attempts')
+ if del_res['success'] and not del_res['action'] == 'bulk_delete':
+ rd = del_res.get('response_dict')
+ if rd is not None:
+ t = dict(rd.get('headers', {}))
+ if t:
+ print(
+ 'Successfully deleted {0}/{1} in {2} attempts '
+ '(transaction id: {3})'.format(c, o, a, t)
+ )
+ else:
+ print(
+ 'Successfully deleted {0}/{1} in {2} '
+ 'attempts'.format(c, o, a)
+ )
diff --git a/examples/download.py b/examples/download.py
new file mode 100644
index 0000000..5e3ebd0
--- /dev/null
+++ b/examples/download.py
@@ -0,0 +1,37 @@
+import logging
+
+from swiftclient.service import SwiftService, SwiftError
+from sys import argv
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+def is_png(obj):
+ return (
+ obj["name"].lower().endswith('.png') or
+ obj["content_type"] == 'image/png'
+ )
+
+container = argv[1]
+with SwiftService() as swift:
+ try:
+ list_options = {"prefix": "archive_2016-01-01/"}
+ list_parts_gen = swift.list(container=container)
+ for page in list_parts_gen:
+ if page["success"]:
+ objects = [
+ obj["name"] for obj in page["listing"] if is_png(obj)
+ ]
+ for down_res in swift.download(
+ container=container,
+ objects=objects):
+ if down_res['success']:
+ print("'%s' downloaded" % down_res['object'])
+ else:
+ print("'%s' download failed" % down_res['object'])
+ else:
+ raise page["error"]
+ except SwiftError as e:
+ logger.error(e.value)
diff --git a/examples/list.py b/examples/list.py
new file mode 100644
index 0000000..4b909d5
--- /dev/null
+++ b/examples/list.py
@@ -0,0 +1,32 @@
+import logging
+
+from swiftclient.service import SwiftService, SwiftError
+from sys import argv
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+container = argv[1]
+minimum_size = 10*1024**2
+with SwiftService() as swift:
+ try:
+ list_parts_gen = swift.list(container=container)
+ for page in list_parts_gen:
+ if page["success"]:
+ for item in page["listing"]:
+
+ i_size = int(item["bytes"])
+ if i_size > minimum_size:
+ i_name = item["name"]
+ i_etag = item["hash"]
+ print(
+ "%s [size: %s] [etag: %s]" %
+ (i_name, i_size, i_etag)
+ )
+ else:
+ raise page["error"]
+
+ except SwiftError as e:
+ logger.error(e.value)
diff --git a/examples/post.py b/examples/post.py
new file mode 100644
index 0000000..c734543
--- /dev/null
+++ b/examples/post.py
@@ -0,0 +1,31 @@
+import logging
+
+from swiftclient.service import SwiftService, SwiftError
+from sys import argv
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+container = argv[1]
+with SwiftService() as swift:
+ try:
+ list_options = {"prefix": "archive_2016-01-01/"}
+ list_parts_gen = swift.list(container=container)
+ for page in list_parts_gen:
+ if page["success"]:
+ objects = [obj["name"] for obj in page["listing"]]
+ post_options = {"header": "X-Delete-After:86400"}
+ for post_res in swift.post(
+ container=container,
+ objects=objects,
+ options=post_options):
+ if post_res['success']:
+ print("Object '%s' POST success" % post_res['object'])
+ else:
+ print("Object '%s' POST failed" % post_res['object'])
+ else:
+ raise page["error"]
+ except SwiftError as e:
+ logger.error(e.value)
diff --git a/examples/stat.py b/examples/stat.py
new file mode 100644
index 0000000..0905d1b
--- /dev/null
+++ b/examples/stat.py
@@ -0,0 +1,25 @@
+import logging
+import pprint
+
+from swiftclient.service import SwiftService
+from sys import argv
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+_opts = {'object_dd_threads': 20}
+with SwiftService(options=_opts) as swift:
+ container = argv[1]
+ objects = argv[2:]
+ header_data = {}
+ stats_it = swift.stat(container=container, objects=objects)
+ for stat_res in stats_it:
+ if stat_res['success']:
+ header_data[stat_res['object']] = stat_res['headers']
+ else:
+ logger.error(
+ 'Failed to retrieve stats for %s' % stat_res['object']
+ )
+ pprint.pprint(header_data)
diff --git a/examples/upload.py b/examples/upload.py
new file mode 100644
index 0000000..1b1e349
--- /dev/null
+++ b/examples/upload.py
@@ -0,0 +1,71 @@
+import logging
+
+from os.path import join, walk
+from swiftclient.multithreading import OutputManager
+from swiftclient.service import SwiftError, SwiftService, SwiftUploadObject
+from sys import argv
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+_opts = {'object_uu_threads': 20}
+dir = argv[1]
+container = argv[2]
+with SwiftService(options=_opts) as swift, OutputManager() as out_manager:
+ try:
+ # Collect all the files and folders in the given directory
+ objs = []
+ dir_markers = []
+ for (_dir, _ds, _fs) in walk(dir):
+ if not (_ds + _fs):
+ dir_markers.append(_dir)
+ else:
+ objs.extend([join(_dir, _f) for _f in _fs])
+
+ # Now that we've collected all the required files and dir markers
+ # build the ``SwiftUploadObject``s for the call to upload
+ objs = [
+ SwiftUploadObject(
+ o, object_name=o.replace(
+ dir, 'my-%s-objects' % dir, 1
+ )
+ ) for o in objs
+ ]
+ dir_markers = [
+ SwiftUploadObject(
+ None, object_name=d.replace(
+ dir, 'my-%s-objects' % dir, 1
+ ), options={'dir_marker': True}
+ ) for d in dir_markers
+ ]
+
+ # Schedule uploads on the SwiftService thread pool and iterate
+ # over the results
+ for r in swift.upload(container, objs + dir_markers):
+ if r['success']:
+ if 'object' in r:
+ print(r['object'])
+ elif 'for_object' in r:
+ print(
+ '%s segment %s' % (r['for_object'],
+ r['segment_index'])
+ )
+ else:
+ error = r['error']
+ if r['action'] == "create_container":
+ logger.warning(
+ 'Warning: failed to create container '
+ "'%s'%s", container, error
+ )
+ elif r['action'] == "upload_object":
+ logger.error(
+ "Failed to upload object %s to container %s: %s" %
+ (container, r['object'], error)
+ )
+ else:
+ logger.error("%s" % error)
+
+ except SwiftError as e:
+ logger.error(e.value)
diff --git a/setup.cfg b/setup.cfg
index 036ef43..5451867 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,7 +5,7 @@ description-file =
README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
-home-page = http://www.openstack.org/
+home-page = http://docs.openstack.org/developer/python-swiftclient
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
diff --git a/swiftclient/client.py b/swiftclient/client.py
index 0726f35..744a876 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -315,6 +315,23 @@ class _RetryBody(_ObjectBody):
response_dict=self.response_dict,
headers=self.headers,
attempts=self.conn.attempts)
+ expected_range = 'bytes %d-%d/%d' % (
+ self.bytes_read,
+ self.expected_length - 1,
+ self.expected_length)
+ if 'content-range' not in hdrs:
+ # Server didn't respond with partial content; manually seek
+ logger.warning('Received 200 while retrying %s/%s; seeking...',
+ self.container, self.obj)
+ to_read = self.bytes_read
+ while to_read > 0:
+ buf = body.resp.read(min(to_read, self.chunk_size))
+ to_read -= len(buf)
+ elif hdrs['content-range'] != expected_range:
+ msg = ('Expected range "%s" while retrying %s/%s '
+ 'but got "%s"' % (expected_range, self.container,
+ self.obj, hdrs['content-range']))
+ raise ClientException(msg)
self.resp = body.resp
buf = self.read(length)
return buf
@@ -682,8 +699,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
:param limit: limit query
:param prefix: prefix query
:param end_marker: end_marker query
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:param service_token: service auth token
@@ -738,8 +755,8 @@ def head_account(url, token, http_conn=None, service_token=None):
:param url: storage URL
:param token: auth token
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param service_token: service auth token
:returns: a dict containing the response's headers (all header names will
be lowercase)
@@ -771,8 +788,8 @@ def post_account(url, token, headers, http_conn=None, response_dict=None,
:param url: storage URL
:param token: auth token
:param headers: additional headers to include in the request
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param response_dict: an optional dictionary into which to place
the response - status, reason and headers
:param service_token: service auth token
@@ -823,8 +840,8 @@ def get_container(url, token, container, marker=None, limit=None,
:param delimiter: string to delimit the queries on
:param end_marker: marker query
:param path: path query (equivalent: "delimiter=/" and "prefix=path/")
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:param service_token: service auth token
@@ -900,8 +917,8 @@ def head_container(url, token, container, http_conn=None, headers=None,
:param url: storage URL
:param token: auth token
:param container: container name to get stats for
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param headers: additional headers to include in the request
:param service_token: service auth token
:returns: a dict containing the response's headers (all header names will
@@ -941,8 +958,8 @@ def put_container(url, token, container, headers=None, http_conn=None,
:param token: auth token
:param container: container name to create
:param headers: additional headers to include in the request
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param response_dict: an optional dictionary into which to place
the response - status, reason and headers
:param service_token: service auth token
@@ -982,8 +999,8 @@ def post_container(url, token, container, headers, http_conn=None,
:param token: auth token
:param container: container name to update
:param headers: additional headers to include in the request
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param response_dict: an optional dictionary into which to place
the response - status, reason and headers
:param service_token: service auth token
@@ -1021,8 +1038,8 @@ def delete_container(url, token, container, http_conn=None,
:param url: storage URL
:param token: auth token
:param container: container name to delete
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param response_dict: an optional dictionary into which to place
the response - status, reason and headers
:param service_token: service auth token
@@ -1060,8 +1077,8 @@ def get_object(url, token, container, name, http_conn=None,
:param token: auth token
:param container: container name that the object is in
:param name: object name to get
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
you specify a resp_chunk_size you must fully read
the object's contents before making another
@@ -1120,8 +1137,8 @@ def head_object(url, token, container, name, http_conn=None,
:param token: auth token
:param container: container name that the object is in
:param name: object name to get info for
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param service_token: service auth token
:param headers: additional headers to include in the request
:returns: a dict containing the response's headers (all header names will
@@ -1182,8 +1199,8 @@ def put_object(url, token=None, container=None, name=None, contents=None,
value is found in the headers param, an empty string
value will be sent
:param headers: additional headers to include in the request, if any
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
:param query_string: if set will be appended with '?' to generated path
@@ -1275,8 +1292,8 @@ def post_object(url, token, container, name, headers, http_conn=None,
:param container: container name that the object is in
:param name: name of the object to update
:param headers: additional headers to include in the request
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param response_dict: an optional dictionary into which to place
the response - status, reason and headers
:param service_token: service auth token
@@ -1314,8 +1331,8 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None,
container name is expected to be part of the url
:param name: object name to delete; if None, the object name is expected to
be part of the url
- :param http_conn: HTTP connection object (If None, it will create the
- conn object)
+ :param http_conn: a tuple of (parsed url, HTTPConnection object),
+ (If None, it will create the conn object)
:param headers: additional headers to include in the request
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
@@ -1360,7 +1377,7 @@ def get_capabilities(http_conn):
"""
Get cluster capability infos.
- :param http_conn: HTTP connection
+ :param http_conn: a tuple of (parsed url, HTTPConnection object)
:returns: a dict containing the cluster capabilities
:raises ClientException: HTTP Capabilities GET failed
"""
diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py
index 2778fac..5e03ed7 100644
--- a/swiftclient/multithreading.py
+++ b/swiftclient/multithreading.py
@@ -126,7 +126,6 @@ class MultiThreadingManager(object):
def __init__(self, create_connection, segment_threads=10,
object_dd_threads=10, object_uu_threads=10,
container_threads=10):
-
"""
:param segment_threads: The number of threads allocated to segment
uploads
diff --git a/swiftclient/service.py b/swiftclient/service.py
index d33d7bc..12d3f21 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -86,12 +86,25 @@ class SwiftError(Exception):
def process_options(options):
- if (not (options.get('auth') and options.get('user')
- and options.get('key'))
- and options.get('auth_version') != '3'):
- # Use keystone 2.0 auth if any of the old-style args are missing
+ # tolerate sloppy auth_version
+ if options.get('auth_version') == '3.0':
+ options['auth_version'] = '3'
+ elif options.get('auth_version') == '2':
options['auth_version'] = '2.0'
+ if options.get('auth_version') not in ('2.0', '3') and not all(
+ options.get(key) for key in ('auth', 'user', 'key')):
+ # Use keystone auth if any of the new-style args are present
+ if any(options.get(k) for k in (
+ 'os_user_domain_id',
+ 'os_user_domain_name',
+ 'os_project_domain_id',
+ 'os_project_domain_name')):
+ # Use v3 if there's any reference to domains
+ options['auth_version'] = '3'
+ else:
+ options['auth_version'] = '2.0'
+
# Use new-style args if old ones not present
if not options['auth'] and options['os_auth_url']:
options['auth'] = options['os_auth_url']
@@ -191,6 +204,10 @@ _default_local_options = {
}
POLICY = 'X-Storage-Policy'
+KNOWN_DIR_MARKERS = (
+ 'application/directory', # Preferred
+ 'text/directory', # Historically relevant
+)
def get_from_queue(q, timeout=864000):
@@ -887,7 +904,7 @@ class SwiftService(object):
@staticmethod
def _list_container_job(conn, container, options, result_queue):
- marker = ''
+ marker = options.get('marker', '')
error = None
try:
while True:
@@ -1134,9 +1151,8 @@ class SwiftService(object):
fp = None
try:
- content_type = headers.get('content-type')
- if (content_type and
- content_type.split(';', 1)[0] == 'text/directory'):
+ content_type = headers.get('content-type', '').split(';', 1)[0]
+ if content_type in KNOWN_DIR_MARKERS:
make_dir = not no_file and out_file != "-"
if make_dir and not isdir(path):
mkdirs(path)
@@ -1594,12 +1610,12 @@ class SwiftService(object):
if options['changed']:
try:
headers = conn.head_object(container, obj)
- ct = headers.get('content-type')
+ ct = headers.get('content-type', '').split(';', 1)[0]
cl = int(headers.get('content-length'))
et = headers.get('etag')
mt = headers.get('x-object-meta-mtime')
- if (ct.split(';', 1)[0] == 'text/directory' and
+ if (ct in KNOWN_DIR_MARKERS and
cl == 0 and
et == EMPTY_ETAG and
mt == put_headers['x-object-meta-mtime']):
@@ -1618,7 +1634,7 @@ class SwiftService(object):
return res
try:
conn.put_object(container, obj, '', content_length=0,
- content_type='text/directory',
+ content_type=KNOWN_DIR_MARKERS[0],
headers=put_headers,
response_dict=results_dict)
res.update({
@@ -1661,10 +1677,13 @@ class SwiftService(object):
fp.seek(segment_start)
contents = LengthWrapper(fp, segment_size, md5=options['checksum'])
- etag = conn.put_object(segment_container,
- segment_name, contents,
- content_length=segment_size,
- response_dict=results_dict)
+ etag = conn.put_object(
+ segment_container,
+ segment_name,
+ contents,
+ content_length=segment_size,
+ content_type='application/swiftclient-segment',
+ response_dict=results_dict)
if options['checksum'] and etag and etag != contents.get_md5sum():
raise SwiftError('Segment {0}: upload verification failed: '
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index c9e1b75..5eafe0b 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -36,7 +36,7 @@ from swiftclient import __version__ as client_version
from swiftclient.client import logger_settings as client_logger_settings, \
parse_header_string
from swiftclient.service import SwiftService, SwiftError, \
- SwiftUploadObject, get_conn
+ SwiftUploadObject, get_conn, process_options
from swiftclient.command_helpers import print_account_stats, \
print_container_stats, print_object_stats
@@ -101,13 +101,13 @@ def st_delete(parser, args, output_manager):
'Its value must be a positive integer. Default is 10.')
(options, args) = parse_args(parser, args)
args = args[1:]
- if (not args and not options.yes_all) or (args and options.yes_all):
+ if (not args and not options['yes_all']) or (args and options['yes_all']):
output_manager.error('Usage: %s delete %s\n%s',
BASENAME, st_delete_options,
st_delete_help)
return
- if options.object_threads <= 0:
+ if options['object_threads'] <= 0:
output_manager.error(
'ERROR: option --object-threads should be a positive integer.'
'\n\nUsage: %s delete %s\n%s',
@@ -115,7 +115,7 @@ def st_delete(parser, args, output_manager):
st_delete_help)
return
- if options.container_threads <= 0:
+ if options['container_threads'] <= 0:
output_manager.error(
'ERROR: option --container-threads should be a positive integer.'
'\n\nUsage: %s delete %s\n%s',
@@ -123,9 +123,8 @@ def st_delete(parser, args, output_manager):
st_delete_help)
return
- _opts = vars(options)
- _opts['object_dd_threads'] = options.object_threads
- with SwiftService(options=_opts) as swift:
+ options['object_dd_threads'] = options['object_threads']
+ with SwiftService(options=options) as swift:
try:
if not args:
del_iter = swift.delete()
@@ -169,7 +168,7 @@ def st_delete(parser, args, output_manager):
pass
for o in objs:
- if options.yes_all:
+ if options['yes_all']:
p = '{0}/{1}'.format(c, o)
else:
p = o
@@ -180,12 +179,12 @@ def st_delete(parser, args, output_manager):
.format(c, o, r['error']))
else:
if r['success']:
- if options.verbose:
+ if options['verbose']:
a = (' [after {0} attempts]'.format(a)
if a > 1 else '')
if r['action'] == 'delete_object':
- if options.yes_all:
+ if options['yes_all']:
p = '{0}/{1}'.format(c, o)
else:
p = o
@@ -203,7 +202,7 @@ def st_delete(parser, args, output_manager):
output_manager.error(err.value)
-st_download_options = '''[--all] [--marker] [--prefix <prefix>]
+st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>]
[--output <out_file>] [--output-dir <out_directory>]
[--object-threads <threads>]
[--container-threads <threads>] [--no-download]
@@ -225,7 +224,7 @@ Positional arguments:
Optional arguments:
-a, --all Indicates that you really want to download
everything in the account.
- -m, --marker Marker to use when starting a container or account
+ -m, --marker <marker> Marker to use when starting a container or account
download.
-p, --prefix <prefix> Only download items beginning with <prefix>
-r, --remove-prefix An optional flag for --prefix <prefix>, use this
@@ -319,40 +318,39 @@ def st_download(parser, args, output_manager):
'are listed in the object store.')
(options, args) = parse_args(parser, args)
args = args[1:]
- if options.out_file == '-':
- options.verbose = 0
+ if options['out_file'] == '-':
+ options['verbose'] = 0
- if options.out_file and len(args) != 2:
+ if options['out_file'] and len(args) != 2:
exit('-o option only allowed for single file downloads')
- if not options.prefix:
- options.remove_prefix = False
+ if not options['prefix']:
+ options['remove_prefix'] = False
- if options.out_directory and len(args) == 2:
+ if options['out_directory'] and len(args) == 2:
exit('Please use -o option for single file downloads and renames')
- if (not args and not options.yes_all) or (args and options.yes_all):
+ if (not args and not options['yes_all']) or (args and options['yes_all']):
output_manager.error('Usage: %s download %s\n%s', BASENAME,
st_download_options, st_download_help)
return
- if options.object_threads <= 0:
+ if options['object_threads'] <= 0:
output_manager.error(
'ERROR: option --object-threads should be a positive integer.\n\n'
'Usage: %s download %s\n%s', BASENAME,
st_download_options, st_download_help)
return
- if options.container_threads <= 0:
+ if options['container_threads'] <= 0:
output_manager.error(
'ERROR: option --container-threads should be a positive integer.'
'\n\nUsage: %s download %s\n%s', BASENAME,
st_download_options, st_download_help)
return
- _opts = vars(options)
- _opts['object_dd_threads'] = options.object_threads
- with SwiftService(options=_opts) as swift:
+ options['object_dd_threads'] = options['object_threads']
+ with SwiftService(options=options) as swift:
try:
if not args:
down_iter = swift.download()
@@ -372,13 +370,13 @@ def st_download(parser, args, output_manager):
down_iter = swift.download(container, objects)
for down in down_iter:
- if options.out_file == '-' and 'contents' in down:
+ if options['out_file'] == '-' and 'contents' in down:
contents = down['contents']
for chunk in contents:
output_manager.print_raw(chunk)
else:
if down['success']:
- if options.verbose:
+ if options['verbose']:
start_time = down['start_time']
headers_receipt = \
down['headers_receipt'] - start_time
@@ -423,7 +421,7 @@ def st_download(parser, args, output_manager):
obj = down['object']
if isinstance(error, ClientException):
if error.http_status == 304 and \
- options.skip_identical:
+ options['skip_identical']:
output_manager.print_msg(
"Skipped identical file '%s'", path)
continue
@@ -467,17 +465,17 @@ Optional arguments:
def st_list(parser, args, output_manager):
- def _print_stats(options, stats):
+ def _print_stats(options, stats, human):
total_count = total_bytes = 0
container = stats.get("container", None)
for item in stats["listing"]:
item_name = item.get('name')
- if not options.long and not options.human:
+ if not options['long'] and not human:
output_manager.print_msg(item.get('name', item.get('subdir')))
else:
if not container: # listing containers
item_bytes = item.get('bytes')
- byte_str = prt_bytes(item_bytes, options.human)
+ byte_str = prt_bytes(item_bytes, human)
count = item.get('count')
total_count += count
try:
@@ -486,7 +484,7 @@ def st_list(parser, args, output_manager):
datestamp = strftime('%Y-%m-%d %H:%M:%S', utc)
except TypeError:
datestamp = '????-??-?? ??:??:??'
- if not options.totals:
+ if not options['totals']:
output_manager.print_msg(
"%5s %s %s %s", count, byte_str,
datestamp, item_name)
@@ -495,29 +493,29 @@ def st_list(parser, args, output_manager):
content_type = item.get('content_type')
if subdir is None:
item_bytes = item.get('bytes')
- byte_str = prt_bytes(item_bytes, options.human)
+ byte_str = prt_bytes(item_bytes, human)
date, xtime = item.get('last_modified').split('T')
xtime = xtime.split('.')[0]
else:
item_bytes = 0
- byte_str = prt_bytes(item_bytes, options.human)
+ byte_str = prt_bytes(item_bytes, human)
date = xtime = ''
item_name = subdir
- if not options.totals:
+ if not options['totals']:
output_manager.print_msg(
"%s %10s %8s %24s %s",
byte_str, date, xtime, content_type, item_name)
total_bytes += item_bytes
# report totals
- if options.long or options.human:
+ if options['long'] or human:
if not container:
output_manager.print_msg(
"%5s %s", prt_bytes(total_count, True),
- prt_bytes(total_bytes, options.human))
+ prt_bytes(total_bytes, human))
else:
output_manager.print_msg(
- prt_bytes(total_bytes, options.human))
+ prt_bytes(total_bytes, human))
parser.add_argument(
'-l', '--long', dest='long', action='store_true', default=False,
@@ -540,20 +538,19 @@ def st_list(parser, args, output_manager):
'what this means.')
options, args = parse_args(parser, args)
args = args[1:]
- if options.delimiter and not args:
+ if options['delimiter'] and not args:
exit('-d option only allowed for container listings')
- _opts = vars(options).copy()
- if _opts['human']:
- _opts.pop('human')
- _opts['long'] = True
+ human = options.pop('human')
+ if human:
+ options['long'] = True
- if options.totals and not options.long and not options.human:
+ if options['totals'] and not options['long']:
output_manager.error(
"Listing totals only works with -l or --lh.")
return
- with SwiftService(options=_opts) as swift:
+ with SwiftService(options=options) as swift:
try:
if not args:
stats_parts_gen = swift.list()
@@ -570,7 +567,7 @@ def st_list(parser, args, output_manager):
for stats in stats_parts_gen:
if stats["success"]:
- _print_stats(options, stats)
+ _print_stats(options, stats, human)
else:
raise stats["error"]
@@ -602,9 +599,7 @@ def st_stat(parser, args, output_manager):
options, args = parse_args(parser, args)
args = args[1:]
- _opts = vars(options)
-
- with SwiftService(options=_opts) as swift:
+ with SwiftService(options=options) as swift:
try:
if not args:
stat_result = swift.stat()
@@ -713,13 +708,11 @@ def st_post(parser, args, output_manager):
'-H "Content-Length: 4000"')
(options, args) = parse_args(parser, args)
args = args[1:]
- if (options.read_acl or options.write_acl or options.sync_to or
- options.sync_key) and not args:
+ if (options['read_acl'] or options['write_acl'] or options['sync_to'] or
+ options['sync_key']) and not args:
exit('-r, -w, -t, and -k options only allowed for containers')
- _opts = vars(options)
-
- with SwiftService(options=_opts) as swift:
+ with SwiftService(options=options) as swift:
try:
if not args:
result = swift.post()
@@ -868,47 +861,46 @@ def st_upload(parser, args, output_manager):
container = args[0]
files = args[1:]
- if options.object_name is not None:
+ if options['object_name'] is not None:
if len(files) > 1:
output_manager.error('object-name only be used with 1 file or dir')
return
else:
orig_path = files[0]
- if options.segment_size:
+ if options['segment_size']:
try:
# If segment size only has digits assume it is bytes
- int(options.segment_size)
+ int(options['segment_size'])
except ValueError:
try:
- size_mod = "BKMG".index(options.segment_size[-1].upper())
- multiplier = int(options.segment_size[:-1])
+ size_mod = "BKMG".index(options['segment_size'][-1].upper())
+ multiplier = int(options['segment_size'][:-1])
except ValueError:
output_manager.error("Invalid segment size")
return
- options.segment_size = str((1024 ** size_mod) * multiplier)
- if int(options.segment_size) <= 0:
+ options['segment_size'] = str((1024 ** size_mod) * multiplier)
+ if int(options['segment_size']) <= 0:
output_manager.error("segment-size should be positive")
return
- if options.object_threads <= 0:
+ if options['object_threads'] <= 0:
output_manager.error(
'ERROR: option --object-threads should be a positive integer.'
'\n\nUsage: %s upload %s\n%s', BASENAME, st_upload_options,
st_upload_help)
return
- if options.segment_threads <= 0:
+ if options['segment_threads'] <= 0:
output_manager.error(
'ERROR: option --segment-threads should be a positive integer.'
'\n\nUsage: %s upload %s\n%s', BASENAME, st_upload_options,
st_upload_help)
return
- _opts = vars(options)
- _opts['object_uu_threads'] = options.object_threads
- with SwiftService(options=_opts) as swift:
+ options['object_uu_threads'] = options['object_threads']
+ with SwiftService(options=options) as swift:
try:
objs = []
dir_markers = []
@@ -926,25 +918,25 @@ def st_upload(parser, args, output_manager):
# Now that we've collected all the required files and dir markers
# build the tuples for the call to upload
- if options.object_name is not None:
+ if options['object_name'] is not None:
objs = [
SwiftUploadObject(
o, object_name=o.replace(
- orig_path, options.object_name, 1
+ orig_path, options['object_name'], 1
)
) for o in objs
]
dir_markers = [
SwiftUploadObject(
None, object_name=d.replace(
- orig_path, options.object_name, 1
+ orig_path, options['object_name'], 1
), options={'dir_marker': True}
) for d in dir_markers
]
for r in swift.upload(container, objs + dir_markers):
if r['success']:
- if options.verbose:
+ if options['verbose']:
if 'attempts' in r and r['attempts'] > 1:
if 'object' in r:
output_manager.print_msg(
@@ -990,7 +982,7 @@ def st_upload(parser, args, output_manager):
output_manager.error("%s" % error)
too_large = (isinstance(error, ClientException) and
error.http_status == 413)
- if too_large and options.verbose > 0:
+ if too_large and options['verbose'] > 0:
output_manager.error(
"Consider using the --segment-size option "
"to chunk the object")
@@ -1028,8 +1020,7 @@ def st_capabilities(parser, args, output_manager):
st_capabilities_options, st_capabilities_help)
return
- _opts = vars(options)
- with SwiftService(options=_opts) as swift:
+ with SwiftService(options=options) as swift:
try:
if len(args) == 2:
url = args[1]
@@ -1067,23 +1058,23 @@ Display auth related authentication variables in shell friendly format.
def st_auth(parser, args, thread_manager):
(options, args) = parse_args(parser, args)
- _opts = vars(options)
- if options.verbose > 1:
- if options.auth_version in ('1', '1.0'):
- print('export ST_AUTH=%s' % sh_quote(options.auth))
- print('export ST_USER=%s' % sh_quote(options.user))
- print('export ST_KEY=%s' % sh_quote(options.key))
+ if options['verbose'] > 1:
+ if options['auth_version'] in ('1', '1.0'):
+ print('export ST_AUTH=%s' % sh_quote(options['auth']))
+ print('export ST_USER=%s' % sh_quote(options['user']))
+ print('export ST_KEY=%s' % sh_quote(options['key']))
else:
print('export OS_IDENTITY_API_VERSION=%s' % sh_quote(
- options.auth_version))
- print('export OS_AUTH_VERSION=%s' % sh_quote(options.auth_version))
- print('export OS_AUTH_URL=%s' % sh_quote(options.auth))
- for k, v in sorted(_opts.items()):
+ options['auth_version']))
+ print('export OS_AUTH_VERSION=%s' % sh_quote(
+ options['auth_version']))
+ print('export OS_AUTH_URL=%s' % sh_quote(options['auth']))
+ for k, v in sorted(options.items()):
if v and k.startswith('os_') and \
k not in ('os_auth_url', 'os_options'):
print('export %s=%s' % (k.upper(), sh_quote(v)))
else:
- conn = get_conn(_opts)
+ conn = get_conn(options)
url, token = conn.get_auth()
print('export OS_STORAGE_URL=%s' % sh_quote(url))
print('export OS_AUTH_TOKEN=%s' % sh_quote(token))
@@ -1140,7 +1131,7 @@ def st_tempurl(parser, args, thread_manager):
'tempurl specified, possibly an error' %
method.upper())
url = generate_temp_url(path, seconds, key, method,
- absolute=options.absolute_expiry)
+ absolute=options['absolute_expiry'])
thread_manager.print_msg(url)
@@ -1179,16 +1170,17 @@ class HelpFormatter(argparse.HelpFormatter):
def parse_args(parser, args, enforce_requires=True):
options, args = parser.parse_known_args(args or ['-h'])
- if enforce_requires and (options.debug or options.info):
+ options = vars(options)
+ if enforce_requires and (options['debug'] or options['info']):
logging.getLogger("swiftclient")
- if options.debug:
+ if options['debug']:
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('iso8601').setLevel(logging.WARNING)
client_logger_settings['redact_sensitive_headers'] = False
- elif options.info:
+ elif options['info']:
logging.basicConfig(level=logging.INFO)
- if args and options.help:
+ if args and options['help']:
_help = globals().get('st_%s_help' % args[0],
"no help for %s" % args[0])
print(_help)
@@ -1198,62 +1190,29 @@ def parse_args(parser, args, enforce_requires=True):
if args and args[0] == 'tempurl':
return options, args
- if options.auth_version == '3.0':
- # tolerate sloppy auth_version
- options.auth_version = '3'
-
- if (not (options.auth and options.user and options.key)
- and options.auth_version != '3'):
- # Use keystone auth if any of the old-style args are missing
- options.auth_version = '2.0'
-
- # Use new-style args if old ones not present
- if not options.auth and options.os_auth_url:
- options.auth = options.os_auth_url
- if not options.user and options.os_username:
- options.user = options.os_username
- if not options.key and options.os_password:
- options.key = options.os_password
-
- # Specific OpenStack options
- options.os_options = {
- 'user_id': options.os_user_id,
- 'user_domain_id': options.os_user_domain_id,
- 'user_domain_name': options.os_user_domain_name,
- 'tenant_id': options.os_tenant_id,
- 'tenant_name': options.os_tenant_name,
- 'project_id': options.os_project_id,
- 'project_name': options.os_project_name,
- 'project_domain_id': options.os_project_domain_id,
- 'project_domain_name': options.os_project_domain_name,
- 'service_type': options.os_service_type,
- 'endpoint_type': options.os_endpoint_type,
- 'auth_token': options.os_auth_token,
- 'object_storage_url': options.os_storage_url,
- 'region_name': options.os_region_name,
- }
+ # Massage auth version; build out os_options subdict
+ process_options(options)
if len(args) > 1 and args[0] == "capabilities":
return options, args
- if (options.os_options.get('object_storage_url') and
- options.os_options.get('auth_token') and
- options.auth_version in ('2.0', '3')):
+ if (options['os_options']['object_storage_url'] and
+ options['os_options']['auth_token']):
return options, args
if enforce_requires:
- if options.auth_version == '3':
- if not options.auth:
+ if options['auth_version'] == '3':
+ if not options['auth']:
exit('Auth version 3 requires OS_AUTH_URL to be set or ' +
'overridden with --os-auth-url')
- if not (options.user or options.os_user_id):
+ if not (options['user'] or options['os_user_id']):
exit('Auth version 3 requires either OS_USERNAME or ' +
'OS_USER_ID to be set or overridden with ' +
'--os-username or --os-user-id respectively.')
- if not options.key:
+ if not options['key']:
exit('Auth version 3 requires OS_PASSWORD to be set or ' +
'overridden with --os-password')
- elif not (options.auth and options.user and options.key):
+ elif not (options['auth'] and options['user'] and options['key']):
exit('''
Auth version 1.0 requires ST_AUTH, ST_USER, and ST_KEY environment variables
to be set or overridden with -A, -U, or -K.
@@ -1549,8 +1508,8 @@ Examples:
'client auth). Defaults to env[OS_KEY].')
options, args = parse_args(parser, argv[1:], enforce_requires=False)
- if options.help or options.os_help:
- if options.help:
+ if options['help'] or options['os_help']:
+ if options['help']:
parser._action_groups.pop()
parser.print_help()
exit()
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py
index e9310aa..cce8c7a 100644
--- a/tests/unit/test_service.py
+++ b/tests/unit/test_service.py
@@ -687,6 +687,41 @@ class TestServiceList(_TestServiceBase):
self.assertEqual(expected_r_long, self._get_queue(mock_q))
self.assertIsNone(self._get_queue(mock_q))
+ def test_list_container_marker(self):
+ mock_q = Queue()
+ mock_conn = self._get_mock_connection()
+
+ get_container_returns = [
+ (None, [{'name': 'b'}, {'name': 'c'}]),
+ (None, [])
+ ]
+ mock_get_cont = Mock(side_effect=get_container_returns)
+ mock_conn.get_container = mock_get_cont
+
+ expected_r = self._get_expected({
+ 'action': 'list_container_part',
+ 'container': 'test_c',
+ 'success': True,
+ 'listing': [{'name': 'b'}, {'name': 'c'}],
+ 'marker': 'b'
+ })
+
+ _opts = self.opts.copy()
+ _opts['marker'] = 'b'
+ SwiftService._list_container_job(mock_conn, 'test_c', _opts, mock_q)
+
+ # This does not test if the marker is propagated, because we always
+ # get the final call to the get_container with the final item 'c',
+ # even if marker wasn't set. This test just makes sure the whole
+ # stack works in a sane way.
+ mock_kw = mock_get_cont.call_args[1]
+ self.assertEqual(mock_kw['marker'], 'c')
+
+ # This tests that the lower levels get the marker delivered.
+ self.assertEqual(expected_r, self._get_queue(mock_q))
+
+ self.assertIsNone(self._get_queue(mock_q))
+
def test_list_container_exception(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
@@ -945,11 +980,12 @@ class TestServiceUpload(_TestServiceBase):
self.assertEqual(r, expected_r)
self.assertEqual(mock_conn.put_object.call_count, 1)
- mock_conn.put_object.assert_called_with('test_c_segments',
- 'test_s_1',
- mock.ANY,
- content_length=10,
- response_dict={})
+ mock_conn.put_object.assert_called_with(
+ 'test_c_segments', 'test_s_1',
+ mock.ANY,
+ content_length=10,
+ content_type='application/swiftclient-segment',
+ response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertIsInstance(contents, utils.LengthWrapper)
self.assertEqual(len(contents), 10)
@@ -988,11 +1024,12 @@ class TestServiceUpload(_TestServiceBase):
self.assertIsNone(r.get('error'))
self.assertEqual(mock_conn.put_object.call_count, 1)
- mock_conn.put_object.assert_called_with('test_c_segments',
- 'test_s_1',
- mock.ANY,
- content_length=10,
- response_dict={})
+ mock_conn.put_object.assert_called_with(
+ 'test_c_segments', 'test_s_1',
+ mock.ANY,
+ content_length=10,
+ content_type='application/swiftclient-segment',
+ response_dict={})
contents = mock_conn.put_object.call_args[0][2]
# Check that md5sum is not calculated.
self.assertEqual(contents.get_md5sum(), '')
@@ -1028,11 +1065,12 @@ class TestServiceUpload(_TestServiceBase):
self.assertIn('md5 mismatch', str(r.get('error')))
self.assertEqual(mock_conn.put_object.call_count, 1)
- mock_conn.put_object.assert_called_with('test_c_segments',
- 'test_s_1',
- mock.ANY,
- content_length=10,
- response_dict={})
+ mock_conn.put_object.assert_called_with(
+ 'test_c_segments', 'test_s_1',
+ mock.ANY,
+ content_length=10,
+ content_type='application/swiftclient-segment',
+ response_dict={})
contents = mock_conn.put_object.call_args[0][2]
self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())
@@ -1348,6 +1386,142 @@ class TestServiceUpload(_TestServiceBase):
errors.append(msg)
self.assertFalse(errors, "\nERRORS:\n%s" % '\n'.join(errors))
+ def test_create_dir_marker_job_unchanged(self):
+ mock_conn = mock.Mock()
+ mock_conn.head_object.return_value = {
+ 'content-type': 'application/directory',
+ 'content-length': '0',
+ 'x-object-meta-mtime': '1.234000',
+ 'etag': md5().hexdigest()}
+
+ s = SwiftService()
+ with mock.patch('swiftclient.service.get_conn',
+ return_value=mock_conn):
+ with mock.patch('swiftclient.service.getmtime',
+ return_value=1.234):
+ r = s._create_dir_marker_job(conn=mock_conn,
+ container='test_c',
+ obj='test_o',
+ path='test',
+ options={'changed': True,
+ 'skip_identical': True,
+ 'leave_segments': True,
+ 'header': '',
+ 'segment_size': 10})
+ self.assertEqual({
+ 'action': 'create_dir_marker',
+ 'container': 'test_c',
+ 'object': 'test_o',
+ 'path': 'test',
+ 'headers': {'x-object-meta-mtime': '1.234000'},
+ # NO response dict!
+ 'success': True,
+ }, r)
+ self.assertEqual([], mock_conn.put_object.mock_calls)
+
+ def test_create_dir_marker_job_unchanged_old_type(self):
+ mock_conn = mock.Mock()
+ mock_conn.head_object.return_value = {
+ 'content-type': 'text/directory',
+ 'content-length': '0',
+ 'x-object-meta-mtime': '1.000000',
+ 'etag': md5().hexdigest()}
+
+ s = SwiftService()
+ with mock.patch('swiftclient.service.get_conn',
+ return_value=mock_conn):
+ with mock.patch('swiftclient.service.time',
+ return_value=1.234):
+ r = s._create_dir_marker_job(conn=mock_conn,
+ container='test_c',
+ obj='test_o',
+ options={'changed': True,
+ 'skip_identical': True,
+ 'leave_segments': True,
+ 'header': '',
+ 'segment_size': 10})
+ self.assertEqual({
+ 'action': 'create_dir_marker',
+ 'container': 'test_c',
+ 'object': 'test_o',
+ 'path': None,
+ 'headers': {'x-object-meta-mtime': '1.000000'},
+ # NO response dict!
+ 'success': True,
+ }, r)
+ self.assertEqual([], mock_conn.put_object.mock_calls)
+
+ def test_create_dir_marker_job_overwrites_bad_type(self):
+ mock_conn = mock.Mock()
+ mock_conn.head_object.return_value = {
+ 'content-type': 'text/plain',
+ 'content-length': '0',
+ 'x-object-meta-mtime': '1.000000',
+ 'etag': md5().hexdigest()}
+
+ s = SwiftService()
+ with mock.patch('swiftclient.service.get_conn',
+ return_value=mock_conn):
+ with mock.patch('swiftclient.service.time',
+ return_value=1.234):
+ r = s._create_dir_marker_job(conn=mock_conn,
+ container='test_c',
+ obj='test_o',
+ options={'changed': True,
+ 'skip_identical': True,
+ 'leave_segments': True,
+ 'header': '',
+ 'segment_size': 10})
+ self.assertEqual({
+ 'action': 'create_dir_marker',
+ 'container': 'test_c',
+ 'object': 'test_o',
+ 'path': None,
+ 'headers': {'x-object-meta-mtime': '1.000000'},
+ 'response_dict': {},
+ 'success': True,
+ }, r)
+ self.assertEqual([mock.call(
+ 'test_c', 'test_o', '',
+ content_length=0,
+ content_type='application/directory',
+ headers={'x-object-meta-mtime': '1.000000'},
+ response_dict={})], mock_conn.put_object.mock_calls)
+
+ def test_create_dir_marker_job_missing(self):
+ mock_conn = mock.Mock()
+ mock_conn.head_object.side_effect = \
+ ClientException('Not Found', http_status=404)
+
+ s = SwiftService()
+ with mock.patch('swiftclient.service.get_conn',
+ return_value=mock_conn):
+ with mock.patch('swiftclient.service.time',
+ return_value=1.234):
+ r = s._create_dir_marker_job(conn=mock_conn,
+ container='test_c',
+ obj='test_o',
+ options={'changed': True,
+ 'skip_identical': True,
+ 'leave_segments': True,
+ 'header': '',
+ 'segment_size': 10})
+ self.assertEqual({
+ 'action': 'create_dir_marker',
+ 'container': 'test_c',
+ 'object': 'test_o',
+ 'path': None,
+ 'headers': {'x-object-meta-mtime': '1.000000'},
+ 'response_dict': {},
+ 'success': True,
+ }, r)
+ self.assertEqual([mock.call(
+ 'test_c', 'test_o', '',
+ content_length=0,
+ content_type='application/directory',
+ headers={'x-object-meta-mtime': '1.000000'},
+ response_dict={})], mock_conn.put_object.mock_calls)
+
class TestServiceDownload(_TestServiceBase):
diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py
index 82a5590..d82def6 100644
--- a/tests/unit/test_shell.py
+++ b/tests/unit/test_shell.py
@@ -765,6 +765,37 @@ class TestShell(unittest.TestCase):
@mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
lambda *a: False)
@mock.patch('swiftclient.service.Connection')
+ def test_delete_bad_threads(self, mock_connection):
+ mock_connection.return_value.get_container.return_value = (None, [])
+ mock_connection.return_value.attempts = 0
+
+ def check_bad(argv):
+ args, env = _make_cmd(
+ 'delete', {}, {}, cmd_args=['cont'] + argv)
+ with mock.patch.dict(os.environ, env):
+ with CaptureOutput() as output:
+ self.assertRaises(SystemExit, swiftclient.shell.main, args)
+ self.assertIn(
+ 'ERROR: option %s should be a positive integer.' % argv[0],
+ output.err)
+
+ def check_good(argv):
+ args, env = _make_cmd(
+ 'delete', {}, {}, cmd_args=['cont'] + argv)
+ with mock.patch.dict(os.environ, env):
+ with CaptureOutput() as output:
+ swiftclient.shell.main(args)
+ self.assertEqual('', output.err)
+ check_bad(["--object-threads", "-1"])
+ check_bad(["--object-threads", "0"])
+ check_bad(["--container-threads", "-1"])
+ check_bad(["--container-threads", "0"])
+ check_good(["--object-threads", "1"])
+ check_good(["--container-threads", "1"])
+
+ @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
+ lambda *a: False)
+ @mock.patch('swiftclient.service.Connection')
def test_delete_account(self, connection):
connection.return_value.get_account.side_effect = [
[None, [{'name': 'container'}, {'name': 'container2'}]],
@@ -1407,18 +1438,18 @@ class TestParsing(TestBase):
expected_os_opts_dict = expected_os_opts_dict or {}
# check the expected opts are set
for key, v in expected_opts.items():
- actual = getattr(actual_opts, key)
+ actual = actual_opts.get(key)
self.assertEqual(v, actual, 'Expected %s for key %s, found %s' %
(v, key, actual))
for key, v in expected_os_opts.items():
- actual = getattr(actual_opts, "os_" + key)
+ actual = actual_opts.get("os_" + key)
self.assertEqual(v, actual, 'Expected %s for key %s, found %s' %
(v, key, actual))
# check the os_options dict values are set
- self.assertTrue(hasattr(actual_opts, 'os_options'))
- actual_os_opts_dict = getattr(actual_opts, 'os_options')
+ self.assertIn('os_options', actual_opts)
+ actual_os_opts_dict = actual_opts['os_options']
expected_os_opts_keys = ['project_name', 'region_name',
'tenant_name',
'user_domain_name', 'endpoint_type',
@@ -1447,8 +1478,8 @@ class TestParsing(TestBase):
('os_auth_url', 'auth'),
('os_password', 'key')]
for pair in equivalents:
- self.assertEqual(getattr(actual_opts, pair[0]),
- getattr(actual_opts, pair[1]))
+ self.assertEqual(actual_opts.get(pair[0]),
+ actual_opts.get(pair[1]))
def test_minimum_required_args_v3(self):
opts = {"auth_version": "3"}
@@ -1491,6 +1522,49 @@ class TestParsing(TestBase):
swiftclient.shell.main(args)
self._verify_opts(result[0], opts, os_opts, os_opts_dict)
+ def test_sloppy_versions(self):
+ os_opts = {"password": "secret",
+ "username": "user",
+ "auth_url": "http://example.com:5000/v3",
+ "identity-api-version": "3.0"}
+
+ # check os_identity_api_version=3.0 is mapped to auth_version=3
+ args = _make_args("stat", {}, os_opts, '-')
+ result = [None, None]
+ fake_command = self._make_fake_command(result)
+ with mock.patch.dict(os.environ, {}):
+ with mock.patch('swiftclient.shell.st_stat', fake_command):
+ swiftclient.shell.main(args)
+ expected_opts = {'auth_version': '3'} # NB: not '3.0'
+ expected_os_opts = {"password": "secret",
+ "username": "user",
+ "auth_url": "http://example.com:5000/v3"}
+ self._verify_opts(result[0], expected_opts, expected_os_opts, {})
+
+ # check os_identity_api_version=2 is mapped to auth_version=2.0
+ # A somewhat contrived scenario - we need to pass in the v1 style opts
+ # to prevent auth version defaulting to 2.0 due to lack of v1 style
+ # options. That way we can actually verify that the sloppy 2 was
+ # interpreted and mapped to 2.0
+ os_opts = {"password": "secret",
+ "username": "user",
+ "auth_url": "http://example.com:5000/v2.0",
+ "identity-api-version": "2"}
+ opts = {"key": "secret",
+ "user": "user",
+ "auth": "http://example.com:5000/v2.0"}
+ args = _make_args("stat", opts, os_opts, '-')
+ result = [None, None]
+ fake_command = self._make_fake_command(result)
+ with mock.patch.dict(os.environ, {}):
+ with mock.patch('swiftclient.shell.st_stat', fake_command):
+ swiftclient.shell.main(args)
+ expected_opts = {'auth_version': '2.0'} # NB: not '2'
+ expected_os_opts = {"password": "secret",
+ "username": "user",
+ "auth_url": "http://example.com:5000/v2.0"}
+ self._verify_opts(result[0], expected_opts, expected_os_opts, {})
+
def test_os_identity_api_version(self):
os_opts = {"password": "secret",
"username": "user",
@@ -1905,6 +1979,13 @@ class TestKeystoneOptions(MockHttpTest):
self._test_options(opts, os_opts, flags=self.flags)
opts = {}
+ self.defaults['auth-version'] = '3'
+ self._test_options(opts, os_opts, flags=self.flags)
+
+ for o in ('user-domain-name', 'user-domain-id',
+ 'project-domain-name', 'project-domain-id'):
+ os_opts.pop(o)
+ self.defaults['auth-version'] = '2.0'
self._test_options(opts, os_opts, flags=self.flags)
def test_catalog_options_and_flags_not_required_v3(self):
@@ -1955,6 +2036,28 @@ class TestKeystoneOptions(MockHttpTest):
os_opts = self._build_os_opts(keys)
self._test_options(opts, os_opts)
+ # ...except when it should be 3
+ self.defaults['auth-version'] = '3'
+ keys = ('username', 'user-domain-name', 'password', 'project-name',
+ 'auth-url')
+ os_opts = self._build_os_opts(keys)
+ self._test_options(opts, os_opts)
+
+ keys = ('username', 'user-domain-id', 'password', 'project-name',
+ 'auth-url')
+ os_opts = self._build_os_opts(keys)
+ self._test_options(opts, os_opts)
+
+ keys = ('username', 'project-domain-name', 'password', 'project-name',
+ 'auth-url')
+ os_opts = self._build_os_opts(keys)
+ self._test_options(opts, os_opts)
+
+ keys = ('username', 'project-domain-id', 'password', 'project-name',
+ 'auth-url')
+ os_opts = self._build_os_opts(keys)
+ self._test_options(opts, os_opts)
+
def test_url_and_token_provided_on_command_line(self):
endpoint = 'http://alternate.com:8080/v1/AUTH_another'
token = 'alternate_auth_token'
@@ -2229,6 +2332,38 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest):
return status
return on_request
+ @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
+ lambda *a: False)
+ @mock.patch('swiftclient.service.Connection')
+ def test_upload_bad_threads(self, mock_connection):
+ mock_connection.return_value.put_object.return_value = EMPTY_ETAG
+ mock_connection.return_value.attempts = 0
+
+ def check_bad(argv):
+ args, env = self._make_cmd(
+ 'upload', cmd_args=[self.cont, self.obj] + argv)
+ with mock.patch.dict(os.environ, env):
+ with CaptureOutput() as output:
+ self.assertRaises(SystemExit, swiftclient.shell.main, args)
+ self.assertIn(
+ 'ERROR: option %s should be a positive integer.' % argv[0],
+ output.err)
+
+ def check_good(argv):
+ args, env = self._make_cmd(
+ 'upload',
+ cmd_args=[self.cont, self.obj, '--leave-segments'] + argv)
+ with mock.patch.dict(os.environ, env):
+ with CaptureOutput() as output:
+ swiftclient.shell.main(args)
+ self.assertEqual('', output.err)
+ check_bad(["--object-threads", "-1"])
+ check_bad(["--object-threads", "0"])
+ check_bad(["--segment-threads", "-1"])
+ check_bad(["--segment-threads", "0"])
+ check_good(["--object-threads", "1"])
+ check_good(["--segment-threads", "1"])
+
def test_upload_with_read_write_access(self):
req_handler = self._fake_cross_account_auth(True, True)
fake_conn = self.fake_http_connection(403, 403,
@@ -2380,6 +2515,38 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest):
self.assertTrue(expected_err in out.err)
self.assertEqual('', out)
+ @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
+ lambda *a: False)
+ @mock.patch('swiftclient.service.Connection')
+ def test_download_bad_threads(self, mock_connection):
+ mock_connection.return_value.get_object.return_value = [{}, '']
+ mock_connection.return_value.attempts = 0
+
+ def check_bad(argv):
+ args, env = self._make_cmd(
+ 'download', cmd_args=[self.cont, self.obj] + argv)
+ with mock.patch.dict(os.environ, env):
+ with CaptureOutput() as output:
+ self.assertRaises(SystemExit, swiftclient.shell.main, args)
+ self.assertIn(
+ 'ERROR: option %s should be a positive integer.' % argv[0],
+ output.err)
+
+ def check_good(argv):
+ args, env = self._make_cmd(
+ 'download',
+ cmd_args=[self.cont, self.obj, '--no-download'] + argv)
+ with mock.patch.dict(os.environ, env):
+ with CaptureOutput() as output:
+ swiftclient.shell.main(args)
+ self.assertEqual('', output.err)
+ check_bad(["--object-threads", "-1"])
+ check_bad(["--object-threads", "0"])
+ check_bad(["--container-threads", "-1"])
+ check_bad(["--container-threads", "0"])
+ check_good(["--object-threads", "1"])
+ check_good(["--container-threads", "1"])
+
def test_download_with_read_write_access(self):
req_handler = self._fake_cross_account_auth(True, True)
fake_conn = self.fake_http_connection(403, on_request=req_handler,
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py
index 5df54b1..cbb95db 100644
--- a/tests/unit/test_swiftclient.py
+++ b/tests/unit/test_swiftclient.py
@@ -952,9 +952,11 @@ class TestGetObject(MockHttpTest):
StubResponse(200, 'abcdef', {'etag': 'some etag',
'content-length': '6'}),
StubResponse(206, 'cdef', {'etag': 'some etag',
- 'content-length': '4'}),
+ 'content-length': '4',
+ 'content-range': 'bytes 2-5/6'}),
StubResponse(206, 'ef', {'etag': 'some etag',
- 'content-length': '2'}),
+ 'content-length': '2',
+ 'content-range': 'bytes 4-5/6'}),
)
__, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2)
self.assertEqual(next(resp), 'ab')
@@ -984,6 +986,70 @@ class TestGetObject(MockHttpTest):
}),
])
+ def test_chunk_size_iter_retry_no_range_support(self):
+ conn = c.Connection('http://auth.url/', 'some_user', 'some_key')
+ with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth:
+ mock_get_auth.return_value = ('http://auth.url', 'tToken')
+ c.http_connection = self.fake_http_connection(*[
+ StubResponse(200, 'abcdef', {'etag': 'some etag',
+ 'content-length': '6'})
+ ] * 3)
+ __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2)
+ self.assertEqual(next(resp), 'ab')
+ self.assertEqual(1, conn.attempts)
+ # simulate a dropped connection
+ resp.resp.read()
+ self.assertEqual(next(resp), 'cd')
+ self.assertEqual(2, conn.attempts)
+ # simulate a dropped connection
+ resp.resp.read()
+ self.assertEqual(next(resp), 'ef')
+ self.assertEqual(3, conn.attempts)
+ self.assertRaises(StopIteration, next, resp)
+ self.assertRequests([
+ ('GET', '/asdf/asdf', '', {
+ 'x-auth-token': 'tToken',
+ }),
+ ('GET', '/asdf/asdf', '', {
+ 'range': 'bytes=2-',
+ 'if-match': 'some etag',
+ 'x-auth-token': 'tToken',
+ }),
+ ('GET', '/asdf/asdf', '', {
+ 'range': 'bytes=4-',
+ 'if-match': 'some etag',
+ 'x-auth-token': 'tToken',
+ }),
+ ])
+
+ def test_chunk_size_iter_retry_bad_range_response(self):
+ conn = c.Connection('http://auth.url/', 'some_user', 'some_key')
+ with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth:
+ mock_get_auth.return_value = ('http://auth.url', 'tToken')
+ c.http_connection = self.fake_http_connection(
+ StubResponse(200, 'abcdef', {'etag': 'some etag',
+ 'content-length': '6'}),
+ StubResponse(206, 'abcdef', {'etag': 'some etag',
+ 'content-length': '6',
+ 'content-range': 'chunk 1-2/3'})
+ )
+ __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2)
+ self.assertEqual(next(resp), 'ab')
+ self.assertEqual(1, conn.attempts)
+ # simulate a dropped connection
+ resp.resp.read()
+ self.assertRaises(c.ClientException, next, resp)
+ self.assertRequests([
+ ('GET', '/asdf/asdf', '', {
+ 'x-auth-token': 'tToken',
+ }),
+ ('GET', '/asdf/asdf', '', {
+ 'range': 'bytes=2-',
+ 'if-match': 'some etag',
+ 'x-auth-token': 'tToken',
+ }),
+ ])
+
def test_get_object_with_resp_chunk_size_zero(self):
def get_connection(self):
def get_auth():
View Lorry Controller Status