From 5ed02345d36acf87fc4678e587db713004696124 Mon Sep 17 00:00:00 2001 From: James Nzomo Date: Sun, 24 Jan 2016 02:43:07 +0300 Subject: Fix segmented upload to pseudo-dir via This fix ensures creation and use of the correct default segment container when pseudo-folder paths are passed via arg. Change-Id: I90356b041dc9dfbd55eb341271975621759476b9 Closes-Bug: 1532981 Related-Bug: 1478210 --- swiftclient/service.py | 25 +++++++++++++------------ tests/unit/test_service.py | 9 +++++++++ tests/unit/test_shell.py | 31 +++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 09245d3..e092bec 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -25,6 +25,7 @@ from os import environ, makedirs, stat, utime from os.path import ( basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep ) +from posixpath import join as urljoin from random import shuffle from time import time from threading import Thread @@ -288,6 +289,7 @@ class SwiftUploadObject(object): if not self.object_name: raise SwiftError('Object names must not be empty strings') + self.object_name = self.object_name.lstrip('/') self.options = options self.source = source @@ -1284,7 +1286,8 @@ class SwiftService(object): """ Upload a list of objects to a given container. - :param container: The container to put the uploads into. + :param container: The container (or pseudo-folder path) to put the + uploads into. :param objects: A list of file/directory names (strings) or SwiftUploadObject instances containing a source for the created object, an object name, and an options dict @@ -1342,10 +1345,9 @@ class SwiftService(object): raise SwiftError('Segment size should be an integer value') # Incase we have a psudeo-folder path for arg, derive - # the container name from the top path to ensure new folder creation - # and prevent spawning zero-byte objects shadowing pseudo-folders - # by name. - container_name = container.split('/', 1)[0] + # the container name from the top path and prepend the rest to + # the object name. (same as passing --object-name). + container, _sep, pseudo_folder = container.partition('/') # Try to create the container, just in case it doesn't exist. If this # fails, it might just be because the user doesn't have container PUT @@ -1358,10 +1360,7 @@ class SwiftService(object): _header[POLICY] create_containers = [ self.thread_manager.container_pool.submit( - self._create_container_job, - container_name, - headers=policy_header - ) + self._create_container_job, container, headers=policy_header) ] # wait for first container job to complete before possibly attempting @@ -1405,7 +1404,7 @@ class SwiftService(object): rq = Queue() file_jobs = {} - upload_objects = self._make_upload_objects(objects) + upload_objects = self._make_upload_objects(objects, pseudo_folder) for upload_object in upload_objects: s = upload_object.source o = upload_object.object_name @@ -1496,14 +1495,16 @@ class SwiftService(object): res = get_from_queue(rq) @staticmethod - def _make_upload_objects(objects): + def _make_upload_objects(objects, pseudo_folder=''): upload_objects = [] for o in objects: if isinstance(o, string_types): - obj = SwiftUploadObject(o) + obj = SwiftUploadObject(o, urljoin(pseudo_folder, + o.lstrip('/'))) upload_objects.append(obj) elif isinstance(o, SwiftUploadObject): + o.object_name = urljoin(pseudo_folder, o.object_name) upload_objects.append(o) else: raise SwiftError( diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 003a51f..c2a7143 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1271,6 +1271,15 @@ class TestServiceUpload(_TestServiceBase): ] mock_conn.get_container.assert_has_calls(expected) + def test_make_upload_objects(self): + # String list + filenames = ['/absolute/file/path', 'relative/file/path'] + self.assertEqual( + [o.object_name for o in SwiftService._make_upload_objects( + filenames, 'pseudo/folder/path')], + ['pseudo/folder/path/absolute/file/path', + 'pseudo/folder/path/relative/file/path']) + class TestServiceDownload(_TestServiceBase): diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 01288c1..1efc8dc 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -485,8 +485,8 @@ class TestShell(testtools.TestCase): response_dict={}) connection.return_value.put_object.assert_called_with( - 'container/pseudo-folder/nested', - self.tmpfile.lstrip('/'), + 'container', + 'pseudo-folder/nested' + self.tmpfile, mock.ANY, content_length=0, headers={'x-object-meta-mtime': mock.ANY, @@ -531,6 +531,33 @@ class TestShell(testtools.TestCase): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + # upload in segments to pseudo-folder (via param) + connection.reset_mock() + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container/pseudo-folder/nested", + self.tmpfile, "-S", "10", "--use-slo"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': 'one'}, + response_dict={})] + connection.return_value.put_container.assert_has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + headers={ + 'x-object-meta-mtime': mock.ANY, + 'x-static-large-object': 'true' + }, + query_string='multipart-manifest=put', + response_dict={}) + @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): argv = ["", "upload", "container", self.tmpfile] -- cgit v1.2.1