diff options
author | Clay Gerrard <clay.gerrard@gmail.com> | 2019-10-29 09:59:03 -0500 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2020-04-08 13:07:26 -0700 |
commit | 78edffa46c591fdc53f253b343e1ea144e24089d (patch) | |
tree | 21205561c27782986e87d60ed1e12176b46d7129 | |
parent | 02e8f4f228c006927fe87f8a350c281b9cfccd98 (diff) | |
download | python-swiftclient-78edffa46c591fdc53f253b343e1ea144e24089d.tar.gz |
object versioning features
* add --versions to list
* add --versions to delete
* add --version-id to stat
* add --version-id to delete
* add --version-id to download
Change-Id: I89802064921778fee7efe57c7d60c976cdde3a27
-rw-r--r-- | swiftclient/client.py | 24 | ||||
-rw-r--r-- | swiftclient/command_helpers.py | 6 | ||||
-rw-r--r-- | swiftclient/service.py | 122 | ||||
-rwxr-xr-x | swiftclient/shell.py | 39 | ||||
-rw-r--r-- | test/unit/test_service.py | 259 | ||||
-rw-r--r-- | test/unit/test_shell.py | 220 |
6 files changed, 611 insertions, 59 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py index 448bb46..449b6cd 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -921,7 +921,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, def get_container(url, token, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, - path=None, http_conn=None, + version_marker=None, path=None, http_conn=None, full_listing=False, service_token=None, headers=None, query_string=None): """ @@ -935,6 +935,7 @@ def get_container(url, token, container, marker=None, limit=None, :param prefix: prefix query :param delimiter: string to delimit the queries on :param end_marker: marker query + :param version_marker: version marker query :param path: path query (equivalent: "delimiter=/" and "prefix=path/") :param http_conn: a tuple of (parsed url, HTTPConnection object), (If None, it will create the conn object) @@ -951,17 +952,20 @@ def get_container(url, token, container, marker=None, limit=None, http_conn = http_connection(url) if full_listing: rv = get_container(url, token, container, marker, limit, prefix, - delimiter, end_marker, path, http_conn, - service_token=service_token, headers=headers) + delimiter, end_marker, version_marker, path=path, + http_conn=http_conn, service_token=service_token, + headers=headers) listing = rv[1] while listing: if not delimiter: marker = listing[-1]['name'] else: marker = listing[-1].get('name', listing[-1].get('subdir')) + version_marker = listing[-1].get('version_id') listing = get_container(url, token, container, marker, limit, - prefix, delimiter, end_marker, path, - http_conn, service_token=service_token, + prefix, delimiter, end_marker, + version_marker, path, http_conn, + service_token=service_token, headers=headers)[1] if listing: rv[1].extend(listing) @@ -979,6 +983,8 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&delimiter=%s' % quote(delimiter) if end_marker: qs += '&end_marker=%s' % quote(end_marker) + if version_marker: + qs += '&version_marker=%s' % quote(version_marker) if path: qs += '&path=%s' % quote(path) if query_string: @@ -1816,15 +1822,17 @@ class Connection(object): return self._retry(None, head_container, container, headers=headers) def get_container(self, container, marker=None, limit=None, prefix=None, - delimiter=None, end_marker=None, path=None, - full_listing=False, headers=None, query_string=None): + delimiter=None, end_marker=None, version_marker=None, + path=None, full_listing=False, headers=None, + query_string=None): """Wrapper for :func:`get_container`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just # retries where it left off. return self._retry(None, get_container, container, marker=marker, limit=limit, prefix=prefix, delimiter=delimiter, - end_marker=end_marker, path=path, + end_marker=end_marker, + version_marker=version_marker, path=path, full_listing=full_listing, headers=headers, query_string=query_string) diff --git a/swiftclient/command_helpers.py b/swiftclient/command_helpers.py index 49ccad1..f37040f 100644 --- a/swiftclient/command_helpers.py +++ b/swiftclient/command_helpers.py @@ -143,7 +143,11 @@ def print_container_stats(items, headers, output_manager): def stat_object(conn, options, container, obj): req_headers = split_request_headers(options.get('header', [])) - headers = conn.head_object(container, obj, headers=req_headers) + query_string = None + if options.get('version_id') is not None: + query_string = 'version-id=%s' % options['version_id'] + headers = conn.head_object(container, obj, headers=req_headers, + query_string=query_string) items = [] if options['verbose'] > 1: path = '%s/%s/%s' % (conn.url, container, obj) diff --git a/swiftclient/service.py b/swiftclient/service.py index 5292dc5..fb334fd 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -86,6 +86,9 @@ class SwiftError(Exception): value += " segment:%s" % self.segment return value + def __repr__(self): + return str(self) + def process_options(options): # tolerate sloppy auth_version @@ -186,6 +189,7 @@ _default_local_options = { 'leave_segments': False, 'changed': None, 'skip_identical': False, + 'version_id': None, 'yes_all': False, 'read_acl': None, 'write_acl': None, @@ -200,6 +204,7 @@ _default_local_options = { 'meta': [], 'prefix': None, 'delimiter': None, + 'versions': False, 'fail_fast': False, 'human': False, 'dir_marker': False, @@ -336,6 +341,20 @@ class SwiftPostObject(object): self.options = options +class SwiftDeleteObject(object): + """ + Class for specifying an object delete, allowing the headers/metadata to be + specified separately for each individual object. + """ + def __init__(self, object_name, options=None): + if not (isinstance(object_name, string_types) and object_name): + raise SwiftError( + "Object names must be specified as non-empty strings" + ) + self.object_name = object_name + self.options = options + + class SwiftCopyObject(object): """ Class for specifying an object copy, @@ -489,6 +508,7 @@ class SwiftService(object): { 'human': False, + 'version_id': None, 'header': [] } @@ -871,6 +891,7 @@ class SwiftService(object): 'long': False, 'prefix': None, 'delimiter': None, + 'versions': False, 'header': [] } @@ -967,13 +988,19 @@ class SwiftService(object): @staticmethod def _list_container_job(conn, container, options, result_queue): marker = options.get('marker', '') + version_marker = options.get('version_marker', '') error = None req_headers = split_headers(options.get('header', [])) + if options.get('versions', False): + query_string = 'versions=true' + else: + query_string = None try: while True: _, items = conn.get_container( - container, marker=marker, prefix=options['prefix'], - delimiter=options['delimiter'], headers=req_headers + container, marker=marker, version_marker=version_marker, + prefix=options['prefix'], delimiter=options['delimiter'], + headers=req_headers, query_string=query_string ) if not items: @@ -991,6 +1018,7 @@ class SwiftService(object): result_queue.put(res) marker = items[-1].get('name', items[-1].get('subdir')) + version_marker = items[-1].get('version_id', '') except ClientException as err: traceback, err_time = report_traceback() logger.exception(err) @@ -1016,6 +1044,7 @@ class SwiftService(object): 'prefix': options['prefix'], 'success': False, 'marker': marker, + 'version_marker': version_marker, 'error': error[0], 'traceback': error[1], 'error_timestamp': error[2] @@ -1042,6 +1071,7 @@ class SwiftService(object): 'no_download': False, 'header': [], 'skip_identical': False, + 'version_id': None, 'out_directory': None, 'checksum': True, 'out_file': None, @@ -1151,6 +1181,9 @@ class SwiftService(object): get_args = {'resp_chunk_size': DISK_BUFFER, 'headers': req_headers, 'response_dict': results_dict} + if options.get('version_id') is not None: + get_args['query_string'] = ( + 'version-id=%s' % options['version_id']) if options['skip_identical']: # Assume the file is a large object; if we're wrong, the query # string is ignored and the If-None-Match header will trigger @@ -2337,14 +2370,28 @@ class SwiftService(object): of objects. :param container: The container to delete or delete from. - :param objects: The list of objects to delete. + :param objects: A list of object names (strings) or SwiftDeleteObject + instances containing an object name, and an + options dict (can be None) to override the options for + that individual delete operation:: + + [ + 'object_name', + SwiftDeleteObject('object_name', + options={...}), + ... + ] + + The options dict is described below. :param options: A dictionary containing options to override the global options specified during the service object creation:: { 'yes_all': False, 'leave_segments': False, + 'version_id': None, 'prefix': None, + 'versions': False, 'header': [], } @@ -2364,23 +2411,28 @@ class SwiftService(object): if container is not None: if objects is not None: + delete_objects = self._make_delete_objects(objects) if options['prefix']: - objects = [obj for obj in objects - if obj.startswith(options['prefix'])] + delete_objects = [ + obj for obj in delete_objects + if obj.object_name.startswith(options['prefix'])] rq = Queue() obj_dels = {} - bulk_page_size = self._bulk_delete_page_size(objects) + bulk_page_size = self._bulk_delete_page_size(delete_objects) if bulk_page_size > 1: - page_at_a_time = n_at_a_time(objects, bulk_page_size) + page_at_a_time = n_at_a_time(delete_objects, + bulk_page_size) for page_slice in page_at_a_time: for obj_slice in n_groups( page_slice, self._options['object_dd_threads']): - self._bulk_delete(container, obj_slice, options, + object_names = [ + obj.object_name for obj in obj_slice] + self._bulk_delete(container, object_names, options, obj_dels) else: - self._per_item_delete(container, objects, options, + self._per_item_delete(container, delete_objects, options, obj_dels, rq) # Start a thread to watch for delete results @@ -2445,6 +2497,11 @@ class SwiftService(object): # Not many objects; may as well delete one-by-one return 1 + if any(obj.options for obj in objects + if isinstance(obj, SwiftDeleteObject)): + # we can't do per option deletes for bulk + return 1 + try: cap_result = self.capabilities() if not cap_result['success']: @@ -2463,9 +2520,11 @@ class SwiftService(object): return 1 def _per_item_delete(self, container, objects, options, rdict, rq): - for obj in objects: + for delete_obj in objects: + obj = delete_obj.object_name + obj_options = dict(options, **delete_obj.options or {}) obj_del = self.thread_manager.object_dd_pool.submit( - self._delete_object, container, obj, options, + self._delete_object, container, obj, obj_options, results_queue=rq ) obj_details = {'container': container, 'object': obj} @@ -2500,6 +2559,24 @@ class SwiftService(object): results_queue.put(res) return res + @staticmethod + def _make_delete_objects(objects): + delete_objects = [] + + for o in objects: + if isinstance(o, string_types): + obj = SwiftDeleteObject(o) + delete_objects.append(obj) + elif isinstance(o, SwiftDeleteObject): + delete_objects.append(o) + else: + raise SwiftError( + "The delete operation takes only strings or " + "SwiftDeleteObjects as input", + obj=o) + + return delete_objects + def _delete_object(self, conn, container, obj, options, results_queue=None): _headers = {} @@ -2511,7 +2588,7 @@ class SwiftService(object): } try: old_manifest = None - query_string = None + query_params = {} if not options['leave_segments']: try: @@ -2520,11 +2597,15 @@ class SwiftService(object): query_string='symlink=get') old_manifest = headers.get('x-object-manifest') if config_true_value(headers.get('x-static-large-object')): - query_string = 'multipart-manifest=delete' + query_params['multipart-manifest'] = 'delete' except ClientException as err: if err.http_status != 404: raise + if options.get('version_id') is not None: + query_params['version-id'] = options['version_id'] + query_string = '&'.join('%s=%s' % (k, v) for (k, v) + in sorted(query_params.items())) results_dict = {} conn.delete_object(container, obj, headers=_headers, @@ -2611,12 +2692,17 @@ class SwiftService(object): try: for part in self.list(container=container, options=options): if not part["success"]: - raise part["error"] - + delete_objects = [] + for item in part['listing']: + delete_opts = {} + if options.get('versions', False) and 'version_id' in item: + delete_opts['version_id'] = item['version_id'] + delete_obj = SwiftDeleteObject(item['name'], delete_opts) + delete_objects.append(delete_obj) for res in self.delete( container=container, - objects=[o['name'] for o in part['listing']], + objects=delete_objects, options=options): yield res if options['prefix']: @@ -2679,7 +2765,9 @@ class SwiftService(object): 'No content received on account POST. ' 'Is the bulk operations middleware enabled?')}) except Exception as e: - res.update({'success': False, 'error': e}) + traceback, err_time = report_traceback() + logger.exception(e) + res.update({'success': False, 'error': e, 'traceback': traceback}) res.update({ 'action': 'bulk_delete', diff --git a/swiftclient/shell.py b/swiftclient/shell.py index d18fc9e..03a8fa6 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -65,7 +65,8 @@ st_delete_options = '''[--all] [--leave-segments] [--container-threads <threads>] [--header <header:value>] [--prefix <prefix>] - [<container> [<object>] [...]] + [--versions] + [<container> [<object>] [--version-id <version_id>] [...]] ''' st_delete_help = ''' @@ -78,6 +79,7 @@ Positional arguments: Optional arguments: -a, --all Delete all containers and objects. + --versions Delete all versions --leave-segments Do not delete segments of manifest objects. -H, --header <header:value> Adds a custom request header to use for deleting @@ -89,6 +91,8 @@ Optional arguments: Number of threads to use for deleting containers. Default is 10. --prefix <prefix> Only delete objects beginning with <prefix>. + --version-id <version-id> + Delete specific version of a versioned object. '''.strip("\n") @@ -96,10 +100,15 @@ def st_delete(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') + parser.add_argument('--versions', action='store_true', + help='delete all versions') parser.add_argument( '-p', '--prefix', dest='prefix', help='Only delete items beginning with <prefix>.') parser.add_argument( + '--version-id', action='store', default=None, + help='Delete a specific version of a versioned object') + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Adds a custom request header to use for deleting objects ' @@ -128,6 +137,10 @@ def st_delete(parser, args, output_manager, return_parser=False): BASENAME, st_delete_options, st_delete_help) return + if options['versions'] and len(args) >= 2: + exit('--versions option not allowed for object deletes') + if options['version_id'] and len(args) < 2: + exit('--version-id option only allowed for object deletes') if options['object_threads'] <= 0: output_manager.error( @@ -227,6 +240,7 @@ st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>] [--object-threads <threads>] [--ignore-checksum] [--container-threads <threads>] [--no-download] [--skip-identical] [--remove-prefix] + [--version-id <version_id>] [--header <header:value>] [--no-shuffle] [<container> [<object>] [...]] ''' @@ -271,6 +285,8 @@ Optional arguments: Example: --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. + --version-id <version-id> + Download specific version of a versioned object. --ignore-checksum Turn off checksum validation for downloads. --no-shuffle By default, when downloading a complete account or container, download order is randomised in order to @@ -333,6 +349,9 @@ def st_download(parser, args, output_manager, return_parser=False): default=False, help='Skip downloading files that are identical on ' 'both sides.') parser.add_argument( + '--version-id', action='store', default=None, + help='Download a specific version of a versioned object') + parser.add_argument( '--ignore-checksum', action='store_false', dest='checksum', default=True, help='Turn off checksum validation for downloads.') parser.add_argument( @@ -372,6 +391,8 @@ def st_download(parser, args, output_manager, return_parser=False): output_manager.error('Usage: %s download %s\n%s', BASENAME, st_download_options, st_download_help) return + if options['version_id'] and len(args) < 2: + exit('--version-id option only allowed for object downloads') if options['object_threads'] <= 0: output_manager.error( @@ -479,7 +500,7 @@ def st_download(parser, args, output_manager, return_parser=False): st_list_options = '''[--long] [--lh] [--totals] [--prefix <prefix>] [--delimiter <delimiter>] [--header <header:value>] - [<container>] + [--versions] [<container>] ''' st_list_help = ''' @@ -499,6 +520,8 @@ Optional arguments: Roll up items with the given delimiter. For containers only. See OpenStack Swift API documentation for what this means. + -j, --json Display listing information in json + --versions Display listing information for all versions -H, --header <header:value> Adds a custom request header to use for listing. '''.strip('\n') @@ -579,6 +602,8 @@ def st_list(parser, args, output_manager, return_parser=False): 'what this means.') parser.add_argument('-j', '--json', action='store_true', help='print listing information in json') + parser.add_argument('--versions', action='store_true', + help='display all versions') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], @@ -592,6 +617,8 @@ def st_list(parser, args, output_manager, return_parser=False): args = args[1:] if options['delimiter'] and not args: exit('-d option only allowed for container listings') + if options['versions'] and not args: + exit('--versions option only allowed for container listings') human = options.pop('human') if human: @@ -642,6 +669,7 @@ def st_list(parser, args, output_manager, return_parser=False): st_stat_options = '''[--lh] [--header <header:value>] + [--version-id <version_id>] [<container> [<object>]] ''' @@ -655,6 +683,8 @@ Positional arguments: Optional arguments: --lh Report sizes in human readable format similar to ls -lh. + --version-id <version-id> + Report stat of specific version of a versioned object. -H, --header <header:value> Adds a custom request header to use for stat. '''.strip('\n') @@ -665,6 +695,9 @@ def st_stat(parser, args, output_manager, return_parser=False): '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') parser.add_argument( + '--version-id', action='store', default=None, + help='Report stat of a specific version of a versioned object') + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Adds a custom request header to use for stat.') @@ -675,6 +708,8 @@ def st_stat(parser, args, output_manager, return_parser=False): options, args = parse_args(parser, args) args = args[1:] + if options['version_id'] and len(args) < 2: + exit('--version-id option only allowed for object stats') with SwiftService(options=options) as swift: try: diff --git a/test/unit/test_service.py b/test/unit/test_service.py index ed3a2d6..e86a4ff 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -21,6 +21,7 @@ import six import tempfile import unittest import time +import json from concurrent.futures import Future from hashlib import md5 @@ -33,7 +34,7 @@ import swiftclient import swiftclient.utils as utils from swiftclient.client import Connection, ClientException from swiftclient.service import ( - SwiftService, SwiftError, SwiftUploadObject + SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject ) from test.unit import utils as test_utils @@ -315,11 +316,39 @@ class TestServiceDelete(_TestServiceBase): mock_conn.head_object.assert_called_once_with( 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, + 'test_c', 'test_o', query_string='', response_dict={}, headers={} ) self.assertEqual(expected_r, r) + @mock.patch('swiftclient.service.Connection') + def test_delete_object_version(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + mock_conn.attempts = 0 + mock_conn.head_object.return_value = {} + mock_conn.delete_object.return_value = {} + expected = { + 'action': 'delete_object', + 'attempts': 0, + 'container': 'c', + 'object': 'o', + 'response_dict': {}, + 'success': True} + with SwiftService() as swift: + delete_results = swift.delete( + container='c', objects='o', options={ + 'version_id': '234567.8'}) + for delete_result in delete_results: + self.assertEqual(delete_result, expected) + self.assertEqual(mock_conn.mock_calls, [ + mock.call.head_object('c', 'o', headers={}, + query_string='symlink=get'), + mock.call.delete_object('c', 'o', headers={}, + query_string='version-id=234567.8', + response_dict={}), + ]) + def test_delete_object_with_headers(self): mock_q = Queue() mock_conn = self._get_mock_connection() @@ -338,7 +367,7 @@ class TestServiceDelete(_TestServiceBase): 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, query_string='symlink=get') mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, + 'test_c', 'test_o', query_string='', response_dict={}, headers={'Skip-Middleware': 'Test'} ) self.assertEqual(expected_r, r) @@ -366,7 +395,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn.head_object.assert_called_once_with( 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, + 'test_c', 'test_o', query_string='', response_dict={}, headers={} ) self.assertEqual(expected_r, r) @@ -431,7 +460,7 @@ class TestServiceDelete(_TestServiceBase): self.assertEqual(expected_r, r) expected = [ - mock.call('test_c', 'test_o', query_string=None, response_dict={}, + mock.call('test_c', 'test_o', query_string='', response_dict={}, headers={}), mock.call('manifest_c', 'test_seg_1', response_dict={}), mock.call('manifest_c', 'test_seg_2', response_dict={})] @@ -529,6 +558,63 @@ class TestServiceDelete(_TestServiceBase): if errors: self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors)) + @mock.patch('swiftclient.service.Connection') + def test_bulk_delete(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.attempts = 0 + mock_conn.get_capabilities.return_value = { + 'bulk_delete': {}} + stub_headers = {} + stub_resp = [] + mock_conn.post_account.return_value = ( + stub_headers, json.dumps(stub_resp).encode('utf8')) + obj_list = ['x%02d' % i for i in range(100)] + expected = [{ + 'action': u'bulk_delete', + 'attempts': 0, + 'container': 'c', + 'objects': list(objs), + 'response_dict': {}, + 'result': [], + 'success': True, + } for objs in zip(*[iter(obj_list)] * 10)] + found_result = [] + with SwiftService(options={'object_dd_threads': 10}) as swift: + delete_results = swift.delete(container='c', objects=obj_list) + for delete_result in delete_results: + found_result.append(delete_result) + self.assertEqual(sorted(found_result, key=lambda r: r['objects'][0]), + expected) + + @mock.patch('swiftclient.service.Connection') + def test_bulk_delete_versions(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.attempts = 0 + mock_conn.get_capabilities.return_value = { + 'bulk_delete': {}} + mock_conn.head_object.return_value = {} + stub_headers = {} + stub_resp = [] + mock_conn.post_account.return_value = ( + stub_headers, json.dumps(stub_resp)) + obj_list = [SwiftDeleteObject('x%02d' % i, options={'version_id': i}) + for i in range(100)] + expected = [{ + 'action': u'delete_object', + 'attempts': 0, + 'container': 'c', + 'object': obj.object_name, + 'response_dict': {}, + 'success': True, + } for obj in obj_list] + found_result = [] + with SwiftService(options={'object_dd_threads': 10}) as swift: + delete_results = swift.delete(container='c', objects=obj_list) + for delete_result in delete_results: + found_result.append(delete_result) + self.assertEqual(sorted(found_result, key=lambda r: r['object']), + expected) + class TestSwiftError(unittest.TestCase): @@ -938,9 +1024,11 @@ class TestServiceList(_TestServiceBase): self.assertIsNone(self._get_queue(mock_q)) self.assertEqual(mock_conn.get_container.mock_calls, [ mock.call('test_c', headers={'Skip-Middleware': 'Test'}, - delimiter='', marker='', prefix=None), + delimiter='', marker='', prefix=None, + query_string=None, version_marker=''), mock.call('test_c', headers={'Skip-Middleware': 'Test'}, - delimiter='', marker='test_o', prefix=None)]) + delimiter='', marker='test_o', prefix=None, + query_string=None, version_marker='')]) def test_list_container_exception(self): mock_q = Queue() @@ -952,6 +1040,7 @@ class TestServiceList(_TestServiceBase): 'success': False, 'error': self.exc, 'marker': '', + 'version_marker': '', 'error_timestamp': mock.ANY, 'traceback': mock.ANY }) @@ -961,11 +1050,61 @@ class TestServiceList(_TestServiceBase): ) mock_conn.get_container.assert_called_once_with( - 'test_c', marker='', delimiter='', prefix=None, headers={} + 'test_c', marker='', delimiter='', prefix=None, headers={}, + query_string=None, version_marker='', ) self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) + @mock.patch('swiftclient.service.Connection') + def test_list_container_versions(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + resp_headers = {} + items = [{ + "bytes": 9, + "content_type": "application/octet-stream", + "hash": "e55cedc11adb39c404b7365f7d6291fa", + "is_latest": True, + "last_modified": "2019-11-08T05:00:15.115360", + "name": "test", + "version_id": "1573189215.11536" + }, { + "bytes": 8, + "content_type": "application/octet-stream", + "hash": "70c1db56f301c9e337b0099bd4174b28", + "is_latest": False, + "last_modified": "2019-11-08T05:00:14.730240", + "name": "test", + "version_id": "1573184903.06720" + }] + mock_conn.get_container.side_effect = [ + (resp_headers, items), + (resp_headers, []), + ] + expected = { + 'action': 'list_container_part', + 'container': 'c', + 'listing': items, + 'marker': '', + 'prefix': None, + 'success': True, + } + with SwiftService() as swift: + list_result_gen = swift.list(container='c', options={ + 'versions': True}) + self.maxDiff = None + for result in list_result_gen: + self.assertEqual(result, expected) + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('c', delimiter=None, headers={}, marker='', + prefix=None, query_string='versions=true', + version_marker=''), + mock.call('c', delimiter=None, headers={}, marker='test', + prefix=None, query_string='versions=true', + version_marker='1573184903.06720'), + ]) + @mock.patch('swiftclient.service.get_conn') def test_list_queue_size(self, mock_get_conn): mock_conn = self._get_mock_connection() @@ -1042,6 +1181,67 @@ class TestServiceList(_TestServiceBase): self.assertEqual(observed_listing, expected_listing) +class TestServiceStat(_TestServiceBase): + + maxDiff = None + + @mock.patch('swiftclient.service.Connection') + def test_stat_object(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + mock_conn.head_object.return_value = {} + expected = { + 'action': 'stat_object', + 'container': 'c', + 'object': 'o', + 'headers': {}, + 'items': [('Account', 'AUTH_test'), + ('Container', 'c'), + ('Object', 'o'), + ('Content Type', None), + ('Content Length', '0'), + ('Last Modified', None), + ('ETag', None), + ('Manifest', None)], + 'success': True} + with SwiftService() as swift: + stat_results = swift.stat(container='c', objects='o') + for stat_result in stat_results: + self.assertEqual(stat_result, expected) + self.assertEqual(mock_conn.head_object.mock_calls, [ + mock.call('c', 'o', headers={}, query_string=None), + ]) + + @mock.patch('swiftclient.service.Connection') + def test_stat_versioned_object(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + mock_conn.head_object.return_value = {} + expected = { + 'action': 'stat_object', + 'container': 'c', + 'object': 'o', + 'headers': {}, + 'items': [('Account', 'AUTH_test'), + ('Container', 'c'), + ('Object', 'o'), + ('Content Type', None), + ('Content Length', '0'), + ('Last Modified', None), + ('ETag', None), + ('Manifest', None)], + 'success': True} + with SwiftService() as swift: + stat_results = swift.stat(container='c', objects='o', options={ + 'version_id': '234567.8'}) + for stat_result in stat_results: + self.assertEqual(stat_result, expected) + self.assertEqual(mock_conn.head_object.mock_calls, [ + mock.call('c', 'o', headers={}, + query_string='version-id=234567.8'), + ]) + + class TestService(unittest.TestCase): def test_upload_with_bad_segment_size(self): @@ -1791,13 +1991,14 @@ class TestServiceUpload(_TestServiceBase): mock_conn.head_object.assert_called_with('test_c', 'test_o') expected = [ mock.call('test_c_segments', prefix='test_o/prefix', - marker='', delimiter=None, headers={}), + marker='', delimiter=None, headers={}, + query_string=None, version_marker=''), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/01", delimiter=None, - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/02", delimiter=None, - headers={}), + headers={}, query_string=None, version_marker=''), ] mock_conn.get_container.assert_has_calls(expected) @@ -2332,6 +2533,29 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual(resp['object'], 'test') self.assertEqual(resp['path'], 'test') + def test_download_version_id(self): + self.opts['version_id'] = '23456.7' + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call( + 'c', 'test', headers={}, query_string='version-id=23456.7', + resp_chunk_size=65536, response_dict={}), + ]) + @mock.patch('swiftclient.service.interruptable_as_completed') @mock.patch('swiftclient.service.SwiftService._download_container') @mock.patch('swiftclient.service.SwiftService._download_object_job') @@ -2545,17 +2769,17 @@ class TestServiceDownload(_TestServiceBase): delimiter=None, prefix='test_o/prefix', marker='', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/2', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/3', - headers={})]) + headers={}, query_string=None, version_marker='')]) def test_download_object_job_skip_identical_nested_slo(self): with tempfile.NamedTemporaryFile() as f: @@ -2682,6 +2906,7 @@ class TestServiceDownload(_TestServiceBase): obj='test_o', options=options) + self.maxDiff = None self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_container.mock_calls, [ @@ -2689,17 +2914,17 @@ class TestServiceDownload(_TestServiceBase): delimiter=None, prefix='test_o/prefix', marker='', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/2', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/3', - headers={})]) + headers={}, query_string=None, version_marker='')]) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 1fa0db4..b94cdcf 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -242,6 +242,30 @@ class TestShell(unittest.TestCase): mock.call('container', headers={'Skip-Middleware': 'Test'})]) @mock.patch('swiftclient.service.Connection') + def test_stat_version_id(self, connection): + argv = ["", "stat", "--version-id", "1"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object stats") + + argv = ["", "stat", "--version-id", "1", "container"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object stats") + + argv = ["", "stat", "--version-id", "1", "container", "object"] + connection.return_value.head_object.return_value = {} + with CaptureOutput(): + swiftclient.shell.main(argv) + self.assertEqual([mock.call('container', 'object', headers={}, + query_string='version-id=1')], + connection.return_value.head_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') def test_stat_object(self, connection): return_headers = { 'x-object-manifest': 'manifest', @@ -295,7 +319,45 @@ class TestShell(unittest.TestCase): ' Manifest: manifest\n') self.assertEqual(connection.return_value.head_object.mock_calls, [ mock.call('container', 'object', - headers={'Skip-Middleware': 'Test'})]) + headers={'Skip-Middleware': 'Test'}, + query_string=None)]) + + def test_list_account_with_delimiter(self): + argv = ["", "list", "--delimiter", "foo"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "-d option only allowed for " + "container listings") + + @mock.patch('swiftclient.service.Connection') + def test_list_container_with_versions(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [ + {'name': 'foo', 'version_id': '2'}, + {'name': 'foo', 'version_id': '1'}, + ]], + [None, []], + ] + argv = ["", "list", "container", "--versions"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call('container', delimiter=None, headers={}, marker='', + prefix=None, query_string='versions=true', + version_marker=''), + mock.call('container', delimiter=None, headers={}, + marker='foo', prefix=None, + query_string='versions=true', version_marker='1')] + connection.return_value.get_container.assert_has_calls(calls) + self.assertEqual(output.out, 'foo\nfoo\n') + + def test_list_account_with_versions(self): + argv = ["", "list", "--versions"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--versions option only allowed for " + "container listings") @mock.patch('swiftclient.service.Connection') def test_list_json(self, connection): @@ -431,9 +493,11 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) calls = [ mock.call('container', marker='', - delimiter=None, prefix=None, headers={}), + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker=''), mock.call('container', marker='object_a', - delimiter=None, prefix=None, headers={})] + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker='')] connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, 'object_a\n') @@ -450,9 +514,11 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) calls = [ mock.call('container', marker='', - delimiter=None, prefix=None, headers={}), + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker=''), mock.call('container', marker='object_a', - delimiter=None, prefix=None, headers={})] + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker='')] connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, @@ -472,14 +538,44 @@ class TestShell(unittest.TestCase): calls = [ mock.call('container', marker='', delimiter=None, prefix=None, - headers={'Skip-Middleware': 'Test'}), + headers={'Skip-Middleware': 'Test'}, + query_string=None, version_marker=''), mock.call('container', marker='object_a', delimiter=None, prefix=None, - headers={'Skip-Middleware': 'Test'})] + headers={'Skip-Middleware': 'Test'}, + query_string=None, version_marker='')] connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, 'object_a\n') + @mock.patch('swiftclient.service.Connection') + def test_download_version_id(self, connection): + argv = ["", "download", "--yes-all", "--version-id", "5"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object downloads") + + argv = ["", "download", "--version-id", "2", "container"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object downloads") + + argv = ["", "download", "--version-id", "1", "container", "object"] + connection.return_value.head_object.return_value = {} + connection.return_value.get_object.return_value = {}, '' + connection.return_value.attempts = 0 + with CaptureOutput(): + swiftclient.shell.main(argv) + self.assertEqual([mock.call('container', 'object', headers={}, + query_string='version-id=1', + resp_chunk_size=65536, + response_dict={})], + connection.return_value.get_object.mock_calls) + @mock.patch('swiftclient.service.makedirs') @mock.patch('swiftclient.service.Connection') def test_download(self, connection, makedirs): @@ -1085,6 +1181,33 @@ class TestShell(unittest.TestCase): check_good(["--object-threads", "1"]) check_good(["--container-threads", "1"]) + @mock.patch('swiftclient.service.Connection') + def test_delete_version_id(self, connection): + argv = ["", "delete", "--yes-all", "--version-id", "3"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object deletes") + + argv = ["", "delete", "--version-id", "1", "container"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object deletes") + + argv = ["", "delete", "--version-id", "1", "container", "object"] + connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None + connection.return_value.attempts = 0 + with CaptureOutput(): + swiftclient.shell.main(argv) + self.assertEqual([mock.call('container', 'object', headers={}, + query_string='version-id=1', + response_dict={})], + connection.return_value.delete_object.mock_calls) + @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') @@ -1094,10 +1217,12 @@ class TestShell(unittest.TestCase): [None, [{'name': 'empty_container'}]], [None, []], ] + # N.B: missing --versions flag, version-id gets ignored + # only latest object is deleted connection.return_value.get_container.side_effect = [ [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], [None, []], - [None, [{'name': 'object'}]], + [None, [{'name': 'object', 'version_id': 1}]], [None, []], [None, []], ] @@ -1107,11 +1232,48 @@ class TestShell(unittest.TestCase): connection.return_value.delete_object.return_value = None swiftclient.shell.main(argv) connection.return_value.delete_object.assert_has_calls([ - mock.call('container', 'object', query_string=None, + mock.call('container', 'object', query_string='', response_dict={}, headers={}), - mock.call('container', 'obj\xe9ct2', query_string=None, + mock.call('container', 'obj\xe9ct2', query_string='', response_dict={}, headers={}), - mock.call('container2', 'object', query_string=None, + mock.call('container2', 'object', query_string='', + response_dict={}, headers={})], any_order=True) + self.assertEqual(3, connection.return_value.delete_object.call_count, + 'Expected 3 calls but found\n%r' + % connection.return_value.delete_object.mock_calls) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_account_versions(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'obj', 'version_id': 1}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--versions"] + connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.delete_object.assert_has_calls([ + mock.call('container', 'object', query_string='', + response_dict={}, headers={}), + mock.call('container', 'obj\xe9ct2', query_string='', + response_dict={}, headers={}), + mock.call('container2', 'obj', query_string='version-id=1', response_dict={}, headers={})], any_order=True) self.assertEqual(3, connection.return_value.delete_object.call_count, 'Expected 3 calls but found\n%r' @@ -1323,12 +1485,42 @@ class TestShell(unittest.TestCase): connection.return_value.delete_container.assert_called_with( 'container', response_dict={}, headers={}) connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, + 'container', 'object', query_string='', response_dict={}, headers={}) @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') + def test_delete_container_versions(self, connection): + argv = ["", "delete", "--versions", "container", "obj"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--versions option not allowed for object deletes") + + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object', 'version_id': 2}, + {'name': 'object', 'version_id': 1}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--versions", "container"] + connection.return_value.head_object.return_value = {} + swiftclient.shell.main(argv) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, headers={}) + expected_calls = [ + mock.call('container', 'object', query_string='version-id=2', + response_dict={}, headers={}), + mock.call('container', 'object', query_string='version-id=1', + response_dict={}, headers={})] + + self.assertEqual(connection.return_value.delete_object.mock_calls, + expected_calls) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') def test_delete_container_headers(self, connection): connection.return_value.get_container.side_effect = [ [None, [{'name': 'object'}]], @@ -1342,7 +1534,7 @@ class TestShell(unittest.TestCase): 'container', response_dict={}, headers={'Skip-Middleware': 'Test'}) connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, + 'container', 'object', query_string='', response_dict={}, headers={'Skip-Middleware': 'Test'}) @mock.patch.object(swiftclient.service.SwiftService, @@ -1408,7 +1600,7 @@ class TestShell(unittest.TestCase): connection.return_value.attempts = 0 swiftclient.shell.main(argv) connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, + 'container', 'object', query_string='', response_dict={}, headers={}) @mock.patch.object(swiftclient.service.SwiftService, |