diff options
author | Tim Burke <tim.burke@gmail.com> | 2015-09-23 10:42:43 -0700 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2016-01-11 11:12:13 -0800 |
commit | d4157ce5b5eeeebb3516092de995cee20025a5c1 (patch) | |
tree | 48105d68f3a7bcf19d67c0593796d10c63eea696 /swiftclient | |
parent | 81003b8d993dc61ef19c6fc4cbe60d9fda76b4d1 (diff) | |
download | python-swiftclient-d4157ce5b5eeeebb3516092de995cee20025a5c1.tar.gz |
Retry file uploads via SwiftService
When we introduced LengthWrapper, we neglected to make it resettable. As
a result, upload failures result in errors like:
put_object(...) failure and no ability to reset contents for reupload.
Now, LengthWrappers will be resettable if their _readable has seek/tell.
Related-Change: I6c8bc1366dfb591a26d934a30cd21c9e6b9a04ce
Change-Id: I21f43f06e8c78b24d1fc081efedf2687942e042f
Diffstat (limited to 'swiftclient')
-rw-r--r-- | swiftclient/client.py | 4 | ||||
-rw-r--r-- | swiftclient/utils.py | 32 |
2 files changed, 31 insertions, 5 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py index aba986e..172f529 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1546,10 +1546,12 @@ class Connection(object): if self.retries > 0: tell = getattr(contents, 'tell', None) seek = getattr(contents, 'seek', None) + reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() reset_func = lambda *a, **k: seek(orig_pos) - + elif reset: + reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, content_length=content_length, etag=etag, chunk_size=chunk_size, content_type=content_type, diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 9d94b6b..ef65bbb 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -202,27 +202,36 @@ class LengthWrapper(object): def __init__(self, readable, length, md5=False): """ :param readable: The filelike object to read from. - :param length: The maximum amount of content to that can be read from + :param length: The maximum amount of content that can be read from the filelike object before it is simulated to be empty. :param md5: Flag to enable calculating the MD5 of the content as it is read. """ - self.md5sum = hashlib.md5() if md5 else NoopMD5() + self._md5 = md5 + self._reset_md5() self._length = self._remaining = length self._readable = readable + self._can_reset = all(hasattr(readable, attr) + for attr in ('seek', 'tell')) + if self._can_reset: + self._start = readable.tell() def __len__(self): return self._length + def _reset_md5(self): + self.md5sum = hashlib.md5() if self._md5 else NoopMD5() + def get_md5sum(self): return self.md5sum.hexdigest() - def read(self, *args, **kwargs): + def read(self, size=-1): if self._remaining <= 0: return '' - chunk = self._readable.read(*args, **kwargs)[:self._remaining] + to_read = self._remaining if size < 0 else min(size, self._remaining) + chunk = self._readable.read(to_read) self._remaining -= len(chunk) try: @@ -232,6 +241,21 @@ class LengthWrapper(object): return chunk + @property + def reset(self): + if self._can_reset: + return self._reset + raise AttributeError("%r object has no attribute 'reset'" % + type(self).__name__) + + def _reset(self, *args, **kwargs): + if not self._can_reset: + raise TypeError('%r object cannot be reset; needs both seek and ' + 'tell methods' % type(self._readable).__name__) + self._readable.seek(self._start) + self._reset_md5() + self._remaining = self._length + def iter_wrapper(iterable): for chunk in iterable: |