diff options
Diffstat (limited to 'swiftclient/service.py')
| -rw-r--r-- | swiftclient/service.py | 52 |
1 files changed, 25 insertions, 27 deletions
diff --git a/swiftclient/service.py b/swiftclient/service.py index 685b748..9a6c7a1 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,9 +12,8 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals -import logging +import logging import os from collections import defaultdict @@ -22,6 +21,7 @@ from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT from hashlib import md5 +from io import StringIO from os import environ, makedirs, stat, utime from os.path import ( basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep @@ -30,10 +30,9 @@ from posixpath import join as urljoin from random import shuffle from time import time from threading import Thread -from six import Iterator, StringIO, string_types, text_type -from six.moves.queue import Queue -from six.moves.queue import Empty as QueueEmpty -from six.moves.urllib.parse import quote +from queue import Queue +from queue import Empty as QueueEmpty +from urllib.parse import quote import json @@ -55,7 +54,7 @@ DISK_BUFFER = 2 ** 16 logger = logging.getLogger("swiftclient.service") -class ResultsIterator(Iterator): +class ResultsIterator: def __init__(self, futures): self.futures = interruptable_as_completed(futures) @@ -156,6 +155,7 @@ def _build_default_global_options(): "user": environ.get('ST_USER'), "key": environ.get('ST_KEY'), "retries": 5, + "retry_on_ratelimit": True, "force_auth_retry": False, "os_username": environ.get('OS_USERNAME'), "os_user_id": environ.get('OS_USER_ID'), @@ -272,9 +272,12 @@ def get_conn(options): """ Return a connection building it from the options. """ + options = dict(_default_global_options, **options) return Connection(options['auth'], options['user'], options['key'], + timeout=options.get('timeout'), + retry_on_ratelimit=options['retry_on_ratelimit'], retries=options['retries'], auth_version=options['auth_version'], os_options=options['os_options'], @@ -317,16 +320,16 @@ def split_headers(options, prefix=''): return headers -class SwiftUploadObject(object): +class SwiftUploadObject: """ Class for specifying an object upload, allowing the object source, name and options to be specified separately for each individual object. """ def __init__(self, source, object_name=None, options=None): - if isinstance(source, string_types): + if isinstance(source, str): self.object_name = object_name or source elif source is None or hasattr(source, 'read'): - if not object_name or not isinstance(object_name, string_types): + if not object_name or not isinstance(object_name, str): raise SwiftError('Object names must be specified as ' 'strings for uploads from None or file ' 'like objects.') @@ -343,13 +346,13 @@ class SwiftUploadObject(object): self.source = source -class SwiftPostObject(object): +class SwiftPostObject: """ Class for specifying an object post, allowing the headers/metadata to be specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -357,13 +360,13 @@ class SwiftPostObject(object): self.options = options -class SwiftDeleteObject(object): +class SwiftDeleteObject: """ Class for specifying an object delete, allowing the headers/metadata to be specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -371,7 +374,7 @@ class SwiftDeleteObject(object): self.options = options -class SwiftCopyObject(object): +class SwiftCopyObject: """ Class for specifying an object copy, allowing the destination/headers/metadata/fresh_metadata to be specified @@ -379,7 +382,7 @@ class SwiftCopyObject(object): destination and fresh_metadata should be set in options """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -407,7 +410,7 @@ class SwiftCopyObject(object): ) -class _SwiftReader(object): +class _SwiftReader: """ Class for downloading objects from swift and raising appropriate errors on failures caused by either invalid md5sum or size of the @@ -472,7 +475,7 @@ class _SwiftReader(object): return self._actual_read -class SwiftService(object): +class SwiftService: """ Service for performing swift operations """ @@ -837,7 +840,7 @@ class SwiftService(object): post_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftPostObject(o) post_objects.append(obj) elif isinstance(o, SwiftPostObject): @@ -1643,7 +1646,7 @@ class SwiftService(object): upload_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftUploadObject(o, urljoin(pseudo_folder, o.lstrip('/'))) upload_objects.append(obj) @@ -2039,11 +2042,6 @@ class SwiftService(object): if headers is None: headers = {} segment_results.sort(key=lambda di: di['segment_index']) - for seg in segment_results: - seg_loc = seg['segment_location'].lstrip('/') - if isinstance(seg_loc, text_type): - seg_loc = seg_loc.encode('utf-8') - manifest_data = json.dumps([ { 'path': d['segment_location'], @@ -2584,7 +2582,7 @@ class SwiftService(object): delete_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftDeleteObject(o) delete_objects.append(obj) elif isinstance(o, SwiftDeleteObject): @@ -2939,7 +2937,7 @@ class SwiftService(object): copy_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftCopyObject(o, options) copy_objects.append(obj) elif isinstance(o, SwiftCopyObject): |