From f1858d89e0e1889664ced654755c508f47a0c1f3 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 11 Jan 2022 16:05:39 -0800 Subject: Add option to skip container PUT during upload Currently, a user with read/write access to a container (but without access to creat new containers) recieves a warning every time they upload. Now, allow them to avoid the extra request and warning by specifying --skip-container-put on the command line. This is also useful when testing: developers can HEAD a container to ensure it's in memcache, shut down all container servers, then upload and creaate a bunch of async pendings. Previously, the 503 on container PUT would prevent the object upload from even being attempted. Closes-Bug: 1317956 Related-Bug: 1204558 Change-Id: I3d9129a0b6b65c6c6187ae6af003b221afceef47 Related-Change: If1f8a02ee7459ea2158ffa6e958f67d299ec529e --- swiftclient/service.py | 95 ++++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 45 deletions(-) (limited to 'swiftclient/service.py') diff --git a/swiftclient/service.py b/swiftclient/service.py index 8e2c7b0..685b748 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -202,6 +202,7 @@ _default_local_options = { 'leave_segments': False, 'changed': None, 'skip_identical': False, + 'skip_container_put': False, 'version_id': None, 'yes_all': False, 'read_acl': None, @@ -1462,6 +1463,7 @@ class SwiftService(object): 'leave_segments': False, 'changed': None, 'skip_identical': False, + 'skip_container_put': False, 'fail_fast': False, 'dir_marker': False # Only for None sources } @@ -1487,54 +1489,57 @@ class SwiftService(object): # the object name. (same as passing --object-name). container, _sep, pseudo_folder = container.partition('/') - # Try to create the container, just in case it doesn't exist. If this - # fails, it might just be because the user doesn't have container PUT - # permissions, so we'll ignore any error. If there's really a problem, - # it'll surface on the first object PUT. - policy_header = {} - _header = split_headers(options["header"]) - if POLICY in _header: - policy_header[POLICY] = \ - _header[POLICY] - create_containers = [ - self.thread_manager.container_pool.submit( - self._create_container_job, container, headers=policy_header) - ] + if not options['skip_container_put']: + # Try to create the container, just in case it doesn't exist. If + # this fails, it might just be because the user doesn't have + # container PUT permissions, so we'll ignore any error. If there's + # really a problem, it'll surface on the first object PUT. + policy_header = {} + _header = split_headers(options["header"]) + if POLICY in _header: + policy_header[POLICY] = \ + _header[POLICY] + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, container, + headers=policy_header) + ] - # wait for first container job to complete before possibly attempting - # segment container job because segment container job may attempt - # to HEAD the first container - for r in interruptable_as_completed(create_containers): - res = r.result() - yield res + # wait for first container job to complete before possibly + # attempting segment container job because segment container job + # may attempt to HEAD the first container + for r in interruptable_as_completed(create_containers): + res = r.result() + yield res - if segment_size: - seg_container = container + '_segments' - if options['segment_container']: - seg_container = options['segment_container'] - if seg_container != container: - if not policy_header: - # Since no storage policy was specified on the command - # line, rather than just letting swift pick the default - # storage policy, we'll try to create the segments - # container with the same policy as the upload container - create_containers = [ - self.thread_manager.container_pool.submit( - self._create_container_job, seg_container, - policy_source=container - ) - ] - else: - create_containers = [ - self.thread_manager.container_pool.submit( - self._create_container_job, seg_container, - headers=policy_header - ) - ] + if segment_size: + seg_container = container + '_segments' + if options['segment_container']: + seg_container = options['segment_container'] + if seg_container != container: + if not policy_header: + # Since no storage policy was specified on the command + # line, rather than just letting swift pick the default + # storage policy, we'll try to create the segments + # container with the same policy as the upload + # container + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, seg_container, + policy_source=container + ) + ] + else: + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, seg_container, + headers=policy_header + ) + ] - for r in interruptable_as_completed(create_containers): - res = r.result() - yield res + for r in interruptable_as_completed(create_containers): + res = r.result() + yield res # We maintain a results queue here and a separate thread to monitor # the futures because we want to get results back from potential -- cgit v1.2.1