diff options
author | Tim Burke <tim.burke@gmail.com> | 2019-04-10 16:22:27 -0700 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2019-06-27 21:45:10 -0700 |
commit | 9021a58c240e156f54ffafdc4609868f348d3ebc (patch) | |
tree | 0ab82d4b2aa595a99cf8d48695a3724f47f3f589 /swiftclient | |
parent | 113eacf3b80f61d366b3e95b558b40f82ff728a4 (diff) | |
download | python-swiftclient-9021a58c240e156f54ffafdc4609868f348d3ebc.tar.gz |
Fix SLO re-upload
Previously, if you uploaded a file as an SLO then re-uploaded it
with the same segment size and mtime, the second upload would
go delete the segments it just (re)uploaded. This was due to
us tracking old_slo_manifest_paths and new_slo_manifest_paths
in different formats; one would have a leading slash while the
other would not.
Now, normalize to the stripped-slash version so we stop deleting
segments we just uploaded.
Change-Id: Ibcbed3df4febe81cdf13855656e2daaca8d521b4
Diffstat (limited to 'swiftclient')
-rw-r--r-- | swiftclient/service.py | 30 | ||||
-rw-r--r-- | swiftclient/utils.py | 8 |
2 files changed, 22 insertions, 16 deletions
diff --git a/swiftclient/service.py b/swiftclient/service.py index 8f3648e..3ea7f61 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -17,6 +17,7 @@ import logging import os +from collections import defaultdict from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -44,7 +45,7 @@ from swiftclient.command_helpers import ( from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, parse_api_response, report_traceback, n_groups, split_request_headers, - n_at_a_time + n_at_a_time, normalize_manifest_path ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager @@ -2071,11 +2072,9 @@ class SwiftService(object): if not options['leave_segments']: old_manifest = headers.get('x-object-manifest') if is_slo: - for old_seg in chunk_data: - seg_path = old_seg['name'].lstrip('/') - if isinstance(seg_path, text_type): - seg_path = seg_path.encode('utf-8') - old_slo_manifest_paths.append(seg_path) + old_slo_manifest_paths.extend( + normalize_manifest_path(old_seg['name']) + for old_seg in chunk_data) except ClientException as err: if err.http_status != 404: traceback, err_time = report_traceback() @@ -2165,8 +2164,9 @@ class SwiftService(object): response = self._upload_slo_manifest( conn, segment_results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - seg['segment_location'] for seg in segment_results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in segment_results) else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2223,8 +2223,9 @@ class SwiftService(object): response = self._upload_slo_manifest( conn, results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - r['segment_location'] for r in results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in results) res['large_object'] = True else: res['response_dict'] = ret @@ -2264,11 +2265,10 @@ class SwiftService(object): fp.close() if old_manifest or old_slo_manifest_paths: drs = [] - delobjsmap = {} + delobjsmap = defaultdict(list) if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) sprefix = sprefix.rstrip('/') + '/' - delobjsmap[scontainer] = [] for part in self.list(scontainer, {'prefix': sprefix}): if not part["success"]: raise part["error"] @@ -2280,10 +2280,8 @@ class SwiftService(object): if seg_to_delete in new_slo_manifest_paths: continue scont, sobj = \ - seg_to_delete.split(b'/', 1) - delobjs_cont = delobjsmap.get(scont, []) - delobjs_cont.append(sobj) - delobjsmap[scont] = delobjs_cont + seg_to_delete.split('/', 1) + delobjsmap[scont].append(sobj) del_segs = [] for dscont, dsobjs in delobjsmap.items(): diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 87a4390..2b208b9 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -395,3 +395,11 @@ def n_at_a_time(seq, n): def n_groups(seq, n): items_per_group = ((len(seq) - 1) // n) + 1 return n_at_a_time(seq, items_per_group) + + +def normalize_manifest_path(path): + if six.PY2 and isinstance(path, six.text_type): + path = path.encode('utf-8') + if path.startswith('/'): + return path[1:] + return path |