diff options
author | Tim Burke <tim.burke@gmail.com> | 2019-04-10 16:22:27 -0700 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2019-08-08 10:36:41 -0700 |
commit | b960279280199f59417ad3e20ca5de11241ef1b0 (patch) | |
tree | a5535efe652d3d326ec04f136c6d1986d7ddd4f1 /swiftclient/service.py | |
parent | 12bd870a02fc3528f45d27152c62fb0b9598bc84 (diff) | |
download | python-swiftclient-b960279280199f59417ad3e20ca5de11241ef1b0.tar.gz |
Fix SLO re-upload
Previously, if you uploaded a file as an SLO then re-uploaded it
with the same segment size and mtime, the second upload would
go delete the segments it just (re)uploaded. This was due to
us tracking old_slo_manifest_paths and new_slo_manifest_paths
in different formats; one would have a leading slash while the
other would not.
Now, normalize to the stripped-slash version so we stop deleting
segments we just uploaded.
Change-Id: Ibcbed3df4febe81cdf13855656e2daaca8d521b4
(cherry picked from commit 9021a58c240e156f54ffafdc4609868f348d3ebc)
(cherry picked from commit 2cd8b86075997f2997118bd92286849ddb0c93d7)
Diffstat (limited to 'swiftclient/service.py')
-rw-r--r-- | swiftclient/service.py | 30 |
1 files changed, 14 insertions, 16 deletions
diff --git a/swiftclient/service.py b/swiftclient/service.py index eedad46..7b63a07 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -17,6 +17,7 @@ import logging import os +from collections import defaultdict from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -44,7 +45,7 @@ from swiftclient.command_helpers import ( from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, parse_api_response, report_traceback, n_groups, split_request_headers, - n_at_a_time + n_at_a_time, normalize_manifest_path ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager @@ -2069,11 +2070,9 @@ class SwiftService(object): if not options['leave_segments']: old_manifest = headers.get('x-object-manifest') if is_slo: - for old_seg in chunk_data: - seg_path = old_seg['name'].lstrip('/') - if isinstance(seg_path, text_type): - seg_path = seg_path.encode('utf-8') - old_slo_manifest_paths.append(seg_path) + old_slo_manifest_paths.extend( + normalize_manifest_path(old_seg['name']) + for old_seg in chunk_data) except ClientException as err: if err.http_status != 404: traceback, err_time = report_traceback() @@ -2163,8 +2162,9 @@ class SwiftService(object): response = self._upload_slo_manifest( conn, segment_results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - seg['segment_location'] for seg in segment_results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in segment_results) else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2221,8 +2221,9 @@ class SwiftService(object): response = self._upload_slo_manifest( conn, results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - r['segment_location'] for r in results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in results) res['large_object'] = True else: res['response_dict'] = ret @@ -2262,11 +2263,10 @@ class SwiftService(object): fp.close() if old_manifest or old_slo_manifest_paths: drs = [] - delobjsmap = {} + delobjsmap = defaultdict(list) if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) sprefix = sprefix.rstrip('/') + '/' - delobjsmap[scontainer] = [] for part in self.list(scontainer, {'prefix': sprefix}): if not part["success"]: raise part["error"] @@ -2278,10 +2278,8 @@ class SwiftService(object): if seg_to_delete in new_slo_manifest_paths: continue scont, sobj = \ - seg_to_delete.split(b'/', 1) - delobjs_cont = delobjsmap.get(scont, []) - delobjs_cont.append(sobj) - delobjsmap[scont] = delobjs_cont + seg_to_delete.split('/', 1) + delobjsmap[scont].append(sobj) del_segs = [] for dscont, dsobjs in delobjsmap.items(): |