summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTim Burke <tim.burke@gmail.com>2019-08-01 21:07:43 -0700
committerTim Burke <tim.burke@gmail.com>2019-08-06 14:14:49 -0700
commit172aa9016977a3b03f07da0350c6aa9ade1e55a0 (patch)
tree2e5209243ad1ddc6f04f6b997d2578dbc9c4b730
parent7d50158440e1ce802cc395135333d2925de11864 (diff)
downloadpython-swiftclient-172aa9016977a3b03f07da0350c6aa9ade1e55a0.tar.gz
Fix up stable gate
This is a combination of three commits from master: --- Isolate docs requirements ...since modern sphinx won't install on py27. While we're at it, clean up some warnings and treat warnings as errors. Also, fix up how we parse test configs so we can run func tests. Related-Change: Id3c2ed87230c5918c18e2c01d086df8157f036b1 (cherry picked from commit 113eacf3b80f61d366b3e95b558b40f82ff728a4) --- Drag forward prettytable in lower-constraints Apparently version 0.7 got unpublished recently. (cherry picked from commit 5bd66947fc3d8987d4b24d5119a346031004229e) --- docs: Clean up formatting (cherry-picked from commit 7563d9cb5642c85a31708fd863e2737f9d6c9419) --- Change-Id: I3c6bca460ee5ff99195495a29e8b8bbf4ed804ff
-rw-r--r--doc/requirements.txt5
-rw-r--r--doc/source/_static/.gitignore0
-rw-r--r--doc/source/cli/index.rst40
-rw-r--r--doc/source/index.rst20
-rw-r--r--doc/source/introduction.rst90
-rw-r--r--doc/source/service-api.rst54
-rw-r--r--lower-constraints.txt2
-rw-r--r--swiftclient/multithreading.py8
-rw-r--r--swiftclient/utils.py2
-rw-r--r--test-requirements.txt3
-rw-r--r--tests/functional/test_swiftclient.py42
-rw-r--r--tox.ini6
12 files changed, 147 insertions, 125 deletions
diff --git a/doc/requirements.txt b/doc/requirements.txt
new file mode 100644
index 0000000..4551673
--- /dev/null
+++ b/doc/requirements.txt
@@ -0,0 +1,5 @@
+keystoneauth1>=3.4.0 # Apache-2.0
+sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD
+sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD
+reno>=2.5.0 # Apache-2.0
+openstackdocstheme>=1.18.1 # Apache-2.0
diff --git a/doc/source/_static/.gitignore b/doc/source/_static/.gitignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/doc/source/_static/.gitignore
diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst
index 88fafa1..d6841d5 100644
--- a/doc/source/cli/index.rst
+++ b/doc/source/cli/index.rst
@@ -245,13 +245,10 @@ storage URL options shown below:
--os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \
list
-.. We need the backslash below in order to indent the note
-\
+.. note::
- .. note::
-
- Leftover environment variables are a common source of confusion when
- authorization fails.
+ Leftover environment variables are a common source of confusion when
+ authorization fails.
CLI commands
~~~~~~~~~~~~
@@ -739,15 +736,15 @@ is passed, the Unix timestamp when the temporary URL will expire.
But beyond that, ``time`` can also be specified as an ISO 8601 timestamp
in one of following formats:
- i) Complete date: YYYY-MM-DD (eg 1997-07-16)
+i) Complete date: YYYY-MM-DD (e.g. 1997-07-16)
- ii) Complete date plus hours, minutes and seconds:
- YYYY-MM-DDThh:mm:ss
- (eg 1997-07-16T19:20:30)
+ii) Complete date plus hours, minutes and seconds:
+ YYYY-MM-DDThh:mm:ss
+ (e.g. 1997-07-16T19:20:30)
- iii) Complete date plus hours, minutes and seconds with UTC designator:
- YYYY-MM-DDThh:mm:ssZ
- (eg 1997-07-16T19:20:30Z)
+iii) Complete date plus hours, minutes and seconds with UTC designator:
+ YYYY-MM-DDThh:mm:ssZ
+ (e.g. 1997-07-16T19:20:30Z)
Please be aware that if you don't provide the UTC designator (i.e., Z)
the timestamp is generated using your local timezone. If only a date is
@@ -881,17 +878,14 @@ Download an object from a container:
testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s]
-.. We need the backslash below in order to indent the note
-\
-
- .. note::
+.. note::
- To upload an object to a container, your current working directory must be
- where the file is located or you must provide the complete path to the file.
- In other words, the --object-name <object-name> is an option that will upload
- file and name object to <object-name> or upload directory and use <object-name> as
- object prefix. In the case that you provide the complete path of the file,
- that complete path will be the name of the uploaded object.
+ To upload an object to a container, your current working directory must be
+ where the file is located or you must provide the complete path to the file.
+ In other words, the --object-name <object-name> is an option that will upload
+ file and name object to <object-name> or upload directory and use <object-name> as
+ object prefix. In the case that you provide the complete path of the file,
+ that complete path will be the name of the uploaded object.
For example:
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 3c2cb1e..ab05c6b 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -39,17 +39,17 @@ Indices and tables
License
~~~~~~~
- Copyright 2013 OpenStack, LLC.
+Copyright 2013 OpenStack, LLC.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+* http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst
index 926b1b9..d6f9819 100644
--- a/doc/source/introduction.rst
+++ b/doc/source/introduction.rst
@@ -16,41 +16,41 @@ Use Cases
Alongside the command line tool, the ``python-swiftclient`` includes two
levels of API:
- * A low level client API that provides simple Python wrappers around the
- various authentication mechanisms and the individual HTTP requests.
- * A high level service API that provides methods for performing common
- operations in parallel on a thread pool.
+* A low level client API that provides simple Python wrappers around the
+ various authentication mechanisms and the individual HTTP requests.
+* A high level service API that provides methods for performing common
+ operations in parallel on a thread pool.
Example use cases:
- * Uploading and retrieving data
- Use the command line tool if you are simply uploading and downloading
- files and directories to and from your filesystem. The command line tool
- can be integrated into a shell script to automate tasks.
-
- * Integrating into an automated Python workflow
- Use the ``SwiftService`` API to perform operations offered by the CLI
- if your use case requires integration with a Python-based workflow.
- This method offers greater control and flexibility over individual object
- operations, such as the metadata set on each object. The ``SwiftService``
- class provides methods to perform multiple sets of operations against a
- swift object store using a configurable shared thread pool. A single
- instance of the ``SwiftService`` class can be shared between multiple
- threads in your own code.
-
- * Developing an application in Python to access a swift object store
- Use the ``SwiftService`` API to develop Python applications that use
- swift to store and retrieve objects. A ``SwiftService`` instance provides
- a configurable thread pool for performing all operations supported by the
- CLI.
-
- * Fine-grained control over threading or the requests being performed
- Use the ``Connection`` API if your use case requires fine grained control
- over advanced features or you wish to use your own existing threading
- model. Examples of advanced features requiring the use of the
- ``Connection`` API include creating an SLO manifest that references
- already existing objects, or fine grained control over the query strings
- supplied with each HTTP request.
+* Uploading and retrieving data
+ Use the command line tool if you are simply uploading and downloading
+ files and directories to and from your filesystem. The command line tool
+ can be integrated into a shell script to automate tasks.
+
+* Integrating into an automated Python workflow
+ Use the ``SwiftService`` API to perform operations offered by the CLI
+ if your use case requires integration with a Python-based workflow.
+ This method offers greater control and flexibility over individual object
+ operations, such as the metadata set on each object. The ``SwiftService``
+ class provides methods to perform multiple sets of operations against a
+ swift object store using a configurable shared thread pool. A single
+ instance of the ``SwiftService`` class can be shared between multiple
+ threads in your own code.
+
+* Developing an application in Python to access a swift object store
+ Use the ``SwiftService`` API to develop Python applications that use
+ swift to store and retrieve objects. A ``SwiftService`` instance provides
+ a configurable thread pool for performing all operations supported by the
+ CLI.
+
+* Fine-grained control over threading or the requests being performed
+ Use the ``Connection`` API if your use case requires fine grained control
+ over advanced features or you wish to use your own existing threading
+ model. Examples of advanced features requiring the use of the
+ ``Connection`` API include creating an SLO manifest that references
+ already existing objects, or fine grained control over the query strings
+ supplied with each HTTP request.
Important considerations
~~~~~~~~~~~~~~~~~~~~~~~~
@@ -66,19 +66,19 @@ the proper use case, and not treat the storage like a traditional filesystem.
There are two main restrictions to bear in mind when designing an application
that uses an object store:
- * You cannot rename objects. Due to fact that the name of an object is one
- of the factors that determines where the object and its replicas are stored,
- renaming would require multiple copies of the data to be moved between
- physical storage devices. If you want to rename an object you must upload
- to the new location, or make a server side copy request to the new location,
- and then delete the original.
-
- * You cannot modify objects. Objects are stored in multiple locations and
- are checked for integrity based on the MD5 sum calculated during
- upload. In order to modify the contents of an object, the entire desired
- contents must be re-uploaded. In certain special cases it is possible to
- work around this restriction using large objects, but no general
- file-like access is available to modify a stored object.
+* You cannot rename objects. Due to fact that the name of an object is one
+ of the factors that determines where the object and its replicas are stored,
+ renaming would require multiple copies of the data to be moved between
+ physical storage devices. If you want to rename an object you must upload
+ to the new location, or make a server side copy request to the new location,
+ and then delete the original.
+
+* You cannot modify objects. Objects are stored in multiple locations and
+ are checked for integrity based on the MD5 sum calculated during
+ upload. In order to modify the contents of an object, the entire desired
+ contents must be re-uploaded. In certain special cases it is possible to
+ work around this restriction using large objects, but no general
+ file-like access is available to modify a stored object.
Objects cannot be locked
------------------------
diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst
index 8efd43a..cc4dcc2 100644
--- a/doc/source/service-api.rst
+++ b/doc/source/service-api.rst
@@ -26,10 +26,10 @@ the auth version based on the combination of options specified, but
supplying options from multiple different auth versions can cause unexpected
behaviour.
- .. note::
+.. note::
- Leftover environment variables are a common source of confusion when
- authorization fails.
+ Leftover environment variables are a common source of confusion when
+ authorization fails.
Keystone V3
~~~~~~~~~~~
@@ -109,17 +109,17 @@ in this dictionary are described below, along with their defaults:
Options
~~~~~~~
- ``retries``: ``5``
+``retries``: ``5``
The number of times that the library should attempt to retry HTTP
actions before giving up and reporting a failure.
- ``container_threads``: ``10``
+``container_threads``: ``10``
- ``object_dd_threads``: ``10``
+``object_dd_threads``: ``10``
- ``object_uu_threads``: ``10``
+``object_uu_threads``: ``10``
- ``segment_threads``: ``10``
+``segment_threads``: ``10``
The above options determine the size of the available thread pools for
performing swift operations. Container operations (such as listing a
container) operate in the container threads, and a similar pattern
@@ -131,86 +131,86 @@ Options
``uu`` and ``dd``. This stands for "upload/update" and "download/delete",
and the corresponding actions will be run on separate threads pools.
- ``segment_size``: ``None``
+``segment_size``: ``None``
If specified, this option enables uploading of large objects. Should the
object being uploaded be larger than 5G in size, this option is
mandatory otherwise the upload will fail. This option should be
specified as a size in bytes.
- ``use_slo``: ``False``
+``use_slo``: ``False``
Used in combination with the above option, ``use_slo`` will upload large
objects as static rather than dynamic. Only static large objects provide
error checking for the downloaded object, so we recommend this option.
- ``segment_container``: ``None``
+``segment_container``: ``None``
Allows the user to select the container into which large object segments
will be uploaded. We do not recommend changing this value as it could make
locating orphaned segments more difficult in the case of errors.
- ``leave_segments``: ``False``
+``leave_segments``: ``False``
Setting this option to true means that when deleting or overwriting a large
object, its segments will be left in the object store and must be cleaned
up manually. This option can be useful when sharing large object segments
between multiple objects in more advanced scenarios, but must be treated
with care, as it could lead to ever increasing storage usage.
- ``changed``: ``None``
+``changed``: ``None``
This option affects uploads and simply means that those objects which
already exist in the object store will not be overwritten if the ``mtime``
and size of the source is the same as the existing object.
- ``skip_identical``: ``False``
+``skip_identical``: ``False``
A slightly more thorough case of the above, but rather than ``mtime`` and size
uses an object's ``MD5 sum``.
- ``yes_all``: ``False``
+``yes_all``: ``False``
This options affects only download and delete, and in each case must be
specified in order to download/delete the entire contents of an account.
This option has no effect on any other calls.
- ``no_download``: ``False``
+``no_download``: ``False``
This option only affects download and means that all operations proceed as
normal with the exception that no data is written to disk.
- ``header``: ``[]``
+``header``: ``[]``
Used with upload and post operations to set headers on objects. Headers
are specified as colon separated strings, e.g. "content-type:text/plain".
- ``meta``: ``[]``
+``meta``: ``[]``
Used to set metadata on an object similarly to headers.
.. note::
Setting metadata is a destructive operation, so when updating one
of many metadata values all desired metadata for an object must be re-applied.
- ``long``: ``False``
+``long``: ``False``
Affects only list operations, and results in more metrics being made
available in the results at the expense of lower performance.
- ``fail_fast``: ``False``
+``fail_fast``: ``False``
Applies to delete and upload operations, and attempts to abort queued
tasks in the event of errors.
- ``prefix``: ``None``
+``prefix``: ``None``
Affects list operations; only objects with the given prefix will be
returned/affected. It is not advisable to set at the service level, as
those operations that call list to discover objects on which they should
operate will also be affected.
- ``delimiter``: ``None``
+``delimiter``: ``None``
Affects list operations, and means that listings only contain results up
to the first instance of the delimiter in the object name. This is useful
for working with objects containing '/' in their names to simulate folder
structures.
- ``dir_marker``: ``False``
+``dir_marker``: ``False``
Affects uploads, and allows empty 'pseudofolder' objects to be created
when the source of an upload is ``None``.
- ``checksum``: ``True``
+``checksum``: ``True``
Affects uploads and downloads. If set check md5 sum for the transfer.
- ``shuffle``: ``False``
+``shuffle``: ``False``
When downloading objects, the default behaviour of the CLI is to shuffle
lists of objects in order to spread the load on storage drives when multiple
clients are downloading the same files to multiple locations (e.g. in the
@@ -220,12 +220,12 @@ Options
are downloaded in lexically-sorted order. Setting this option to ``True``
gives the same shuffling behaviour as the CLI.
- ``destination``: ``None``
+``destination``: ``None``
When copying objects, this specifies the destination where the object
will be copied to. The default of None means copy will be the same as
source.
- ``fresh_metadata``: ``None``
+``fresh_metadata``: ``None``
When copying objects, this specifies that the object metadata on the
source will *not* be applied to the destination object - the
destination object will have a new fresh set of metadata that includes
diff --git a/lower-constraints.txt b/lower-constraints.txt
index fefb90a..aa995cc 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -25,7 +25,7 @@ oslo.config==1.2.0
oslosphinx==4.7.0
pbr==2.0.0
pep8==1.5.7
-PrettyTable==0.7
+PrettyTable==0.7.1
pyflakes==0.8.1
Pygments==2.2.0
python-keystoneclient==0.7.0
diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py
index 5e03ed7..fcf0ed9 100644
--- a/swiftclient/multithreading.py
+++ b/swiftclient/multithreading.py
@@ -175,6 +175,14 @@ class ConnectionThreadPoolExecutor(ThreadPoolExecutor):
super(ConnectionThreadPoolExecutor, self).__init__(max_workers)
def submit(self, fn, *args, **kwargs):
+ """
+ Schedules the callable, `fn`, to be executed
+
+ :param fn: the callable to be invoked
+ :param args: the positional arguments for the callable
+ :param kwargs: the keyword arguments for the callable
+ :returns: a Future object representing the execution of the callable
+ """
def conn_fn():
priority = None
conn = None
diff --git a/swiftclient/utils.py b/swiftclient/utils.py
index 5c17c61..87a4390 100644
--- a/swiftclient/utils.py
+++ b/swiftclient/utils.py
@@ -74,7 +74,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False,
Swift object.
:param path: The full path to the Swift object or prefix if
- a prefix-based temporary URL should be generated. Example:
+ a prefix-based temporary URL should be generated. Example:
/v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix.
:param seconds: time in seconds or ISO 8601 timestamp.
If absolute is False and this is the string representation of an
diff --git a/test-requirements.txt b/test-requirements.txt
index 13cf1e9..ca65bca 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -4,7 +4,4 @@ coverage!=4.4,>=4.0 # Apache-2.0
keystoneauth1>=3.4.0 # Apache-2.0
mock>=1.2.0 # BSD
oslosphinx>=4.7.0 # Apache-2.0
-sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
stestr>=2.0.0 # Apache-2.0
-reno>=2.5.0 # Apache-2.0
-openstackdocstheme>=1.18.1 # Apache-2.0
diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py
index b4f275b..bae3044 100644
--- a/tests/functional/test_swiftclient.py
+++ b/tests/functional/test_swiftclient.py
@@ -46,11 +46,34 @@ class TestFunctional(unittest.TestCase):
config.read(config_file)
self.config = config
if config.has_section('func_test'):
- auth_host = config.get('func_test', 'auth_host')
- auth_port = config.getint('func_test', 'auth_port')
- auth_ssl = config.getboolean('func_test', 'auth_ssl')
- auth_prefix = config.get('func_test', 'auth_prefix')
- self.auth_version = config.get('func_test', 'auth_version')
+ if config.has_option('func_test', 'auth_uri'):
+ self.auth_url = config.get('func_test', 'auth_uri')
+ try:
+ self.auth_version = config.get('func_test', 'auth_version')
+ except configparser.NoOptionError:
+ last_piece = self.auth_url.rstrip('/').rsplit('/', 1)[1]
+ if last_piece.endswith('.0'):
+ last_piece = last_piece[:-2]
+ if last_piece in ('1', '2', '3'):
+ self.auth_version = last_piece
+ else:
+ raise
+ else:
+ auth_host = config.get('func_test', 'auth_host')
+ auth_port = config.getint('func_test', 'auth_port')
+ auth_ssl = config.getboolean('func_test', 'auth_ssl')
+ auth_prefix = config.get('func_test', 'auth_prefix')
+ self.auth_version = config.get('func_test', 'auth_version')
+ self.auth_url = ""
+ if auth_ssl:
+ self.auth_url += "https://"
+ else:
+ self.auth_url += "http://"
+ self.auth_url += "%s:%s%s" % (
+ auth_host, auth_port, auth_prefix)
+ if self.auth_version == "1":
+ self.auth_url += 'v1.0'
+
try:
self.account_username = config.get('func_test',
'account_username')
@@ -59,15 +82,6 @@ class TestFunctional(unittest.TestCase):
username = config.get('func_test', 'username')
self.account_username = "%s:%s" % (account, username)
self.password = config.get('func_test', 'password')
- self.auth_url = ""
- if auth_ssl:
- self.auth_url += "https://"
- else:
- self.auth_url += "http://"
- self.auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix)
- if self.auth_version == "1":
- self.auth_url += 'v1.0'
-
else:
self.skip_tests = True
diff --git a/tox.ini b/tox.ini
index 613e3f8..0018432 100644
--- a/tox.ini
+++ b/tox.ini
@@ -65,8 +65,10 @@ commands = {[testenv:func]commands}
[testenv:docs]
basepython = python3
+usedevelop = False
+deps = -r{toxinidir}/doc/requirements.txt
commands=
- python setup.py build_sphinx
+ python setup.py build_sphinx -W
[flake8]
# it's not a bug that we aren't using all of hacking, ignore:
@@ -93,6 +95,8 @@ commands = bindep test
[testenv:releasenotes]
basepython = python3
+usedevelop = False
+deps = -r{toxinidir}/doc/requirements.txt
commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
[testenv:lower-constraints]