summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarek Kaleta <Marek.Kaleta@firma.seznam.cz>2016-02-15 12:14:17 +0100
committerTim Burke <tim.burke@gmail.com>2016-08-23 14:37:11 -0700
commit4a2465fb12ff7287b62b6941fb8ae43e100adc25 (patch)
tree1b01f6d6bd044019f671fea1808e87b3646982d9
parente05464fbfa03ee6d938c145ede111f4b1e828d58 (diff)
downloadpython-swiftclient-4a2465fb12ff7287b62b6941fb8ae43e100adc25.tar.gz
Add copy object method
Implement copy object method in swiftclient Connection, Service and CLI. Although COPY functionality can be accomplished with 'X-Copy-From' header in PUT request, using copy is more convenient especially when using copy for updating object metadata non-destructively. Closes-Bug: 1474939 Change-Id: I1338ac411f418f4adb3d06753d044a484a7f32a4
-rw-r--r--doc/manpages/swift.113
-rw-r--r--doc/source/cli.rst14
-rw-r--r--doc/source/service-api.rst82
-rw-r--r--examples/copy.py30
-rw-r--r--swiftclient/client.py71
-rw-r--r--swiftclient/service.py225
-rwxr-xr-xswiftclient/shell.py102
-rw-r--r--tests/functional/test_swiftclient.py40
-rw-r--r--tests/unit/test_service.py106
-rw-r--r--tests/unit/test_shell.py133
-rw-r--r--tests/unit/test_swiftclient.py104
11 files changed, 914 insertions, 6 deletions
diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1
index b9f99c4..7c2ee89 100644
--- a/doc/manpages/swift.1
+++ b/doc/manpages/swift.1
@@ -79,6 +79,19 @@ For more details and options see swift post \-\-help.
\fBExample\fR: post \-m Color:Blue \-m Size:Large
.RE
+\fBcopy\fR [\fIcommand-options\fR] \fIcontainer\fR \fIobject\fR
+.RS 4
+Copies an object to a new destination or adds user metadata to the object (current
+user metadata will be preserved, in contrast with the post command) depending
+on the args given. The \-\-destination option sets the destination in the form
+/container/object. If not set, the object will be copied onto itself which is useful
+for adding metadata. The \-M or \-\-fresh\-metadata option copies the object without
+the existing user metadata. The \-m or \-\-meta option is always allowed and is used
+to define the user metadata items to set in the form Name:Value (this option
+can be repeated).
+For more details and options see swift copy \-\-help.
+.RE
+
\fBdownload\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...]
.RS 4
Downloads everything in the account (with \-\-all), or everything in a
diff --git a/doc/source/cli.rst b/doc/source/cli.rst
index 12de02f..76630be 100644
--- a/doc/source/cli.rst
+++ b/doc/source/cli.rst
@@ -199,6 +199,20 @@ Delete
of manifest objects will be deleted as well, unless you specify the
``--leave-segments`` option.
+Copy
+----
+
+ ``copy [command-options] container object``
+
+ Copies an object to a new destination or adds user metadata to an object. Depending
+ on the options supplied, you can preserve existing metadata in contrast to the post
+ command. The ``--destination`` option sets the copy target destination in the form
+ ``/container/object``. If not set, the object will be copied onto itself which is useful
+ for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy
+ an object without existing user meta data, and the ``-m`` or ``--meta`` option
+ to define user meta data items to set in the form ``Name:Value``. You can repeat
+ this option. For example: ``copy -m Color:Blue -m Size:Large``.
+
Capabilities
------------
diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst
index 50dd80e..aeb3daa 100644
--- a/doc/source/service-api.rst
+++ b/doc/source/service-api.rst
@@ -217,6 +217,17 @@ Options
are downloaded in lexically-sorted order. Setting this option to ``True``
gives the same shuffling behaviour as the CLI.
+ ``destination``: ``None``
+ When copying objects, this specifies the destination where the object
+ will be copied to. The default of None means copy will be the same as
+ source.
+
+ ``fresh_metadata``: ``None``
+ When copying objects, this specifies that the object metadata on the
+ source will *not* be applied to the destination object - the
+ destination object will have a new fresh set of metadata that includes
+ *only* the metadata specified in the meta option if any at all.
+
Other available options can be found in ``swiftclient/service.py`` in the
source code for ``python-swiftclient``. Each ``SwiftService`` method also allows
for an optional dictionary to override those specified at init time, and the
@@ -735,13 +746,76 @@ Example
The code below demonstrates the use of ``delete`` to remove a given list of
objects from a specified container. As the objects are deleted the transaction
-id of the relevant request is printed along with the object name and number
-of attempts required. By printing the transaction id, the printed operations
+ID of the relevant request is printed along with the object name and number
+of attempts required. By printing the transaction ID, the printed operations
can be easily linked to events in the swift server logs:
.. literalinclude:: ../../examples/delete.py
:language: python
+Copy
+~~~~
+
+Copy can be called to copy an object or update the metadata on the given items.
+
+Each element of the object list may be a plain string of the object name, or a
+``SwiftCopyObject`` that allows finer control over the options applied to each
+of the individual copy operations (destination, fresh_metadata, options).
+
+Destination should be in format /container/object; if not set, the object will be
+copied onto itself. Fresh_metadata sets mode of operation on metadata. If not set,
+current object user metadata will be copied/preserved; if set, all current user
+metadata will be removed.
+
+Returns an iterator over the results generated for each object copy (and may
+also include the results of creating destination containers).
+
+When a string is given for the object name, destination and fresh metadata will
+default to None and None, which result in adding metadata to existing objects.
+
+Successful copy results are dictionaries as described below:
+
+.. code-block:: python
+
+ {
+ 'action': 'copy_object',
+ 'success': True,
+ 'container': <container>,
+ 'object': <object>,
+ 'destination': <destination>,
+ 'headers': {},
+ 'fresh_metadata': <boolean>,
+ 'response_dict': <HTTP response details>
+ }
+
+Any failure in a copy operation will return a failure dictionary as described
+below:
+
+.. code-block:: python
+
+ {
+ 'action': 'copy_object',
+ 'success': False,
+ 'container': <container>,
+ 'object': <object>,
+ 'destination': <destination>,
+ 'headers': {},
+ 'fresh_metadata': <boolean>,
+ 'response_dict': <HTTP response details>,
+ 'error': <error>,
+ 'traceback': <traceback>,
+ 'error_timestamp': <timestamp>
+ }
+
+Example
+-------
+
+The code below demonstrates the use of ``copy`` to add new user metadata for
+objects a and b, and to copy object c to d (with added metadata).
+
+.. literalinclude:: ../../examples/copy.py
+ :language: python
+
Capabilities
~~~~~~~~~~~~
@@ -764,7 +838,7 @@ returned with the contents described below:
}
The contents of the capabilities dictionary contain the core swift capabilities
-under the key ``swift``, all other keys show the configuration options for
+under the key ``swift``; all other keys show the configuration options for
additional middlewares deployed in the proxy pipeline. An example capabilities
dictionary is given below:
@@ -819,7 +893,7 @@ Example
^^^^^^^
The code below demonstrates the use of ``capabilities`` to determine if the
-Swift cluster supports static large objects, and if so, the maximum number of
+Swift cluster supports static large objects, and if so, the maximum number of
segments that can be described in a single manifest file, along with the
size restrictions on those objects:
diff --git a/examples/copy.py b/examples/copy.py
new file mode 100644
index 0000000..e928db4
--- /dev/null
+++ b/examples/copy.py
@@ -0,0 +1,30 @@
+import logging
+
+from swiftclient.service import SwiftService, SwiftCopyObject, SwiftError
+
+logging.basicConfig(level=logging.ERROR)
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
+logger = logging.getLogger(__name__)
+
+with SwiftService() as swift:
+ try:
+ obj = SwiftCopyObject("c", {"Destination": "/cont/d"})
+ for i in swift.copy(
+ "cont", ["a", "b", obj],
+ {"meta": ["foo:bar"], "Destination": "/cc"}):
+ if i["success"]:
+ if i["action"] == "copy_object":
+ print(
+ "object %s copied from /%s/%s" %
+ (i["destination"], i["container"], i["object"])
+ )
+ if i["action"] == "create_container":
+ print(
+ "container %s created" % i["container"]
+ )
+ else:
+ if "error" in i and isinstance(i["error"], Exception):
+ raise i["error"]
+ except SwiftError as e:
+ logger.error(e.value)
diff --git a/swiftclient/client.py b/swiftclient/client.py
index 744a876..5b3fa72 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -1319,6 +1319,70 @@ def post_object(url, token, container, name, headers, http_conn=None,
raise ClientException.from_response(resp, 'Object POST failed', body)
+def copy_object(url, token, container, name, destination=None,
+ headers=None, fresh_metadata=None, http_conn=None,
+ response_dict=None, service_token=None):
+ """
+ Copy object
+
+ :param url: storage URL
+ :param token: auth token; if None, no token will be sent
+ :param container: container name that the source object is in
+ :param name: source object name
+ :param destination: The container and object name of the destination object
+ in the form of /container/object; if None, the copy
+ will use the source as the destination.
+ :param headers: additional headers to include in the request
+ :param fresh_metadata: Enables object creation that omits existing user
+ metadata, default None
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :param response_dict: an optional dictionary into which to place
+ the response - status, reason and headers
+ :param service_token: service auth token
+ :raises ClientException: HTTP COPY request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+
+ path = parsed.path
+ container = quote(container)
+ name = quote(name)
+ path = '%s/%s/%s' % (path.rstrip('/'), container, name)
+
+ headers = dict(headers) if headers else {}
+
+ if destination is not None:
+ headers['Destination'] = quote(destination)
+ elif container and name:
+ headers['Destination'] = '/%s/%s' % (container, name)
+
+ if token is not None:
+ headers['X-Auth-Token'] = token
+ if service_token is not None:
+ headers['X-Service-Token'] = service_token
+
+ if fresh_metadata is not None:
+ # remove potential fresh metadata headers
+ for fresh_hdr in [hdr for hdr in headers.keys()
+ if hdr.lower() == 'x-fresh-metadata']:
+ headers.pop(fresh_hdr)
+ headers['X-Fresh-Metadata'] = 'true' if fresh_metadata else 'false'
+
+ conn.request('COPY', path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'COPY',),
+ {'headers': headers}, resp, body)
+
+ store_response(resp, response_dict)
+
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException.from_response(resp, 'Object COPY failed', body)
+
+
def delete_object(url, token=None, container=None, name=None, http_conn=None,
headers=None, proxy=None, query_string=None,
response_dict=None, service_token=None):
@@ -1711,6 +1775,13 @@ class Connection(object):
return self._retry(None, post_object, container, obj, headers,
response_dict=response_dict)
+ def copy_object(self, container, obj, destination=None, headers=None,
+ fresh_metadata=None, response_dict=None):
+ """Wrapper for :func:`copy_object`"""
+ return self._retry(None, copy_object, container, obj, destination,
+ headers, fresh_metadata,
+ response_dict=response_dict)
+
def delete_object(self, container, obj, query_string=None,
response_dict=None):
"""Wrapper for :func:`delete_object`"""
diff --git a/swiftclient/service.py b/swiftclient/service.py
index 12d3f21..a813d16 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -200,7 +200,9 @@ _default_local_options = {
'human': False,
'dir_marker': False,
'checksum': True,
- 'shuffle': False
+ 'shuffle': False,
+ 'destination': None,
+ 'fresh_metadata': False,
}
POLICY = 'X-Storage-Policy'
@@ -330,6 +332,42 @@ class SwiftPostObject(object):
self.options = options
+class SwiftCopyObject(object):
+ """
+ Class for specifying an object copy,
+ allowing the destination/headers/metadata/fresh_metadata to be specified
+ separately for each individual object.
+ destination and fresh_metadata should be set in options
+ """
+ def __init__(self, object_name, options=None):
+ if not isinstance(object_name, string_types) or not object_name:
+ raise SwiftError(
+ "Object names must be specified as non-empty strings"
+ )
+
+ self.object_name = object_name
+ self.options = options
+
+ if self.options is None:
+ self.destination = None
+ self.fresh_metadata = False
+ else:
+ self.destination = self.options.get('destination')
+ self.fresh_metadata = self.options.get('fresh_metadata', False)
+
+ if self.destination is not None:
+ destination_components = self.destination.split('/')
+ if destination_components[0] or len(destination_components) < 2:
+ raise SwiftError("destination must be in format /cont[/obj]")
+ if not destination_components[-1]:
+ raise SwiftError("destination must not end in a slash")
+ if len(destination_components) == 2:
+ # only container set in destination
+ self.destination = "{0}/{1}".format(
+ self.destination, object_name
+ )
+
+
class _SwiftReader(object):
"""
Class for downloading objects from swift and raising appropriate
@@ -2389,6 +2427,191 @@ class SwiftService(object):
return res
+ # Copy related methods
+ #
+ def copy(self, container, objects, options=None):
+ """
+ Copy operations on a list of objects in a container. Destination
+ containers will be created.
+
+ :param container: The container from which to copy the objects.
+ :param objects: A list of object names (strings) or SwiftCopyObject
+ instances containing an object name and an
+ options dict (can be None) to override the options for
+ that individual copy operation::
+
+ [
+ 'object_name',
+ SwiftCopyObject(
+ 'object_name',
+ options={
+ 'destination': '/container/object',
+ 'fresh_metadata': False,
+ ...
+ }),
+ ...
+ ]
+
+ The options dict is described below.
+ :param options: A dictionary containing options to override the global
+ options specified during the service object creation.
+ These options are applied to all copy operations
+ performed by this call, unless overridden on a per
+ object basis.
+ The options "destination" and "fresh_metadata" do
+ not need to be set, in this case objects will be
+ copied onto themselves and metadata will not be
+ refreshed.
+ The option "destination" can also be specified in the
+ format '/container', in which case objects without an
+ explicit destination will be copied to the destination
+ /container/original_object_name. Combinations of
+ multiple objects and a destination in the format
+ '/container/object' is invalid. Possible options are
+ given below::
+
+ {
+ 'meta': [],
+ 'header': [],
+ 'destination': '/container/object',
+ 'fresh_metadata': False,
+ }
+
+ :returns: A generator returning the results of copying the given list
+ of objects.
+
+ :raises: SwiftError
+ """
+ if options is not None:
+ options = dict(self._options, **options)
+ else:
+ options = self._options
+
+ # Try to create the container, just in case it doesn't exist. If this
+ # fails, it might just be because the user doesn't have container PUT
+ # permissions, so we'll ignore any error. If there's really a problem,
+ # it'll surface on the first object COPY.
+ containers = set(
+ next(p for p in obj.destination.split("/") if p)
+ for obj in objects
+ if isinstance(obj, SwiftCopyObject) and obj.destination
+ )
+ if options.get('destination'):
+ destination_split = options['destination'].split('/')
+ if destination_split[0]:
+ raise SwiftError("destination must be in format /cont[/obj]")
+ _str_objs = [
+ o for o in objects if not isinstance(o, SwiftCopyObject)
+ ]
+ if len(destination_split) > 2 and len(_str_objs) > 1:
+ # TODO (clayg): could be useful to copy multiple objects into
+ # a destination like "/container/common/prefix/for/objects/"
+ # where the trailing "/" indicates the destination option is a
+ # prefix!
+ raise SwiftError("Combination of multiple objects and "
+ "destination including object is invalid")
+ if destination_split[-1] == '':
+ # N.B. this protects the above case
+ raise SwiftError("destination can not end in a slash")
+ containers.add(destination_split[1])
+
+ policy_header = {}
+ _header = split_headers(options["header"])
+ if POLICY in _header:
+ policy_header[POLICY] = _header[POLICY]
+ create_containers = [
+ self.thread_manager.container_pool.submit(
+ self._create_container_job, cont, headers=policy_header)
+ for cont in containers
+ ]
+
+ # wait for container creation jobs to complete before any COPY
+ for r in interruptable_as_completed(create_containers):
+ res = r.result()
+ yield res
+
+ copy_futures = []
+ copy_objects = self._make_copy_objects(objects, options)
+ for copy_object in copy_objects:
+ obj = copy_object.object_name
+ obj_options = copy_object.options
+ destination = copy_object.destination
+ fresh_metadata = copy_object.fresh_metadata
+ headers = split_headers(
+ options['meta'], 'X-Object-Meta-')
+ # add header options to the headers object for the request.
+ headers.update(
+ split_headers(options['header'], ''))
+ if obj_options is not None:
+ if 'meta' in obj_options:
+ headers.update(
+ split_headers(
+ obj_options['meta'], 'X-Object-Meta-'
+ )
+ )
+ if 'header' in obj_options:
+ headers.update(
+ split_headers(obj_options['header'], '')
+ )
+
+ copy = self.thread_manager.object_uu_pool.submit(
+ self._copy_object_job, container, obj, destination,
+ headers, fresh_metadata
+ )
+ copy_futures.append(copy)
+
+ for r in interruptable_as_completed(copy_futures):
+ res = r.result()
+ yield res
+
+ @staticmethod
+ def _make_copy_objects(objects, options):
+ copy_objects = []
+
+ for o in objects:
+ if isinstance(o, string_types):
+ obj = SwiftCopyObject(o, options)
+ copy_objects.append(obj)
+ elif isinstance(o, SwiftCopyObject):
+ copy_objects.append(o)
+ else:
+ raise SwiftError(
+ "The copy operation takes only strings or "
+ "SwiftCopyObjects as input",
+ obj=o)
+
+ return copy_objects
+
+ @staticmethod
+ def _copy_object_job(conn, container, obj, destination, headers,
+ fresh_metadata):
+ response_dict = {}
+ res = {
+ 'success': True,
+ 'action': 'copy_object',
+ 'container': container,
+ 'object': obj,
+ 'destination': destination,
+ 'headers': headers,
+ 'fresh_metadata': fresh_metadata,
+ 'response_dict': response_dict
+ }
+ try:
+ conn.copy_object(
+ container, obj, destination=destination, headers=headers,
+ fresh_metadata=fresh_metadata, response_dict=response_dict)
+ except Exception as err:
+ traceback, err_time = report_traceback()
+ logger.exception(err)
+ res.update({
+ 'success': False,
+ 'error': err,
+ 'traceback': traceback,
+ 'error_timestamp': err_time
+ })
+
+ return res
+
# Capabilities related methods
#
def capabilities(self, url=None, refresh_cache=False):
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index 5eafe0b..8da0e7b 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -46,7 +46,7 @@ except ImportError:
from pipes import quote as sh_quote
BASENAME = 'swift'
-commands = ('delete', 'download', 'list', 'post', 'stat', 'upload',
+commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload',
'capabilities', 'info', 'tempurl', 'auth')
@@ -746,6 +746,105 @@ def st_post(parser, args, output_manager):
output_manager.error(e.value)
+st_copy_options = '''[--destination </container/object>] [--fresh-metadata]
+ [--meta <name:value>] [--header <header>] container object
+'''
+
+st_copy_help = '''
+Copies object to new destination, optionally updates objects metadata.
+If destination is not set, will update metadata of object
+
+Positional arguments:
+ container Name of container to copy from.
+ object Name of object to copy. Specify multiple times
+ for multiple objects
+
+Optional arguments:
+ -d, --destination </container[/object]>
+ The container and name of the destination object. Name
+ of destination object can be ommited, then will be
+ same as name of source object. Supplying multiple
+ objects and destination with object name is invalid.
+ -M, --fresh-metadata Copy the object without any existing metadata,
+ If not set, metadata will be preserved or appended
+ -m, --meta <name:value>
+ Sets a meta data item. This option may be repeated.
+ Example: -m Color:Blue -m Size:Large
+ -H, --header <header:value>
+ Adds a customized request header.
+ This option may be repeated. Example
+ -H "content-type:text/plain" -H "Content-Length: 4000"
+'''.strip('\n')
+
+
+def st_copy(parser, args, output_manager):
+ parser.add_argument(
+ '-d', '--destination', help='The container and name of the '
+ 'destination object')
+ parser.add_argument(
+ '-M', '--fresh-metadata', action='store_true',
+ help='Copy the object without any existing metadata', default=False)
+ parser.add_argument(
+ '-m', '--meta', action='append', dest='meta', default=[],
+ help='Sets a meta data item. This option may be repeated. '
+ 'Example: -m Color:Blue -m Size:Large')
+ parser.add_argument(
+ '-H', '--header', action='append', dest='header',
+ default=[], help='Adds a customized request header. '
+ 'This option may be repeated. '
+ 'Example: -H "content-type:text/plain" '
+ '-H "Content-Length: 4000"')
+ (options, args) = parse_args(parser, args)
+ args = args[1:]
+
+ with SwiftService(options=options) as swift:
+ try:
+ if len(args) >= 2:
+ container = args[0]
+ if '/' in container:
+ output_manager.error(
+ 'WARNING: / in container name; you might have '
+ "meant '%s' instead of '%s'." %
+ (args[0].replace('/', ' ', 1), args[0]))
+ return
+ objects = [arg for arg in args[1:]]
+
+ for r in swift.copy(
+ container=container, objects=objects,
+ options=options):
+ if r['success']:
+ if options['verbose']:
+ if r['action'] == 'copy_object':
+ output_manager.print_msg(
+ '%s/%s copied to %s' % (
+ r['container'],
+ r['object'],
+ r['destination'] or '<self>'))
+ if r['action'] == 'create_container':
+ output_manager.print_msg(
+ 'created container %s' % r['container']
+ )
+ else:
+ error = r['error']
+ if 'action' in r and r['action'] == 'create_container':
+ # it is not an error to be unable to create the
+ # container so print a warning and carry on
+ output_manager.warning(
+ 'Warning: failed to create container '
+ "'%s': %s", container, error
+ )
+ else:
+ output_manager.error("%s" % error)
+ else:
+ output_manager.error(
+ 'Usage: %s copy %s\n%s', BASENAME,
+ st_copy_options, st_copy_help)
+ return
+
+ except SwiftError as e:
+ output_manager.error(e.value)
+
+
st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
[--segment-container <container>] [--leave-segments]
[--object-threads <thread>] [--segment-threads <threads>]
@@ -1268,6 +1367,7 @@ Positional arguments:
for a container.
post Updates meta information for the account, container,
or object; creates containers if not present.
+ copy Copies object, optionally adds meta
stat Displays information for the account, container,
or object.
upload Uploads files or directories to the given container.
diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py
index 7a77c07..6e19abd 100644
--- a/tests/functional/test_swiftclient.py
+++ b/tests/functional/test_swiftclient.py
@@ -405,6 +405,46 @@ class TestFunctional(unittest.TestCase):
headers = self.conn.head_object(self.containername, self.objectname)
self.assertEqual('Something', headers.get('x-object-meta-color'))
+ def test_copy_object(self):
+ self.conn.put_object(
+ self.containername, self.objectname, self.test_data)
+ self.conn.copy_object(self.containername,
+ self.objectname,
+ headers={'x-object-meta-color': 'Something'})
+
+ headers = self.conn.head_object(self.containername, self.objectname)
+ self.assertEqual('Something', headers.get('x-object-meta-color'))
+
+ self.conn.copy_object(self.containername,
+ self.objectname,
+ headers={'x-object-meta-taste': 'Second'})
+
+ headers = self.conn.head_object(self.containername, self.objectname)
+ self.assertEqual('Something', headers.get('x-object-meta-color'))
+ self.assertEqual('Second', headers.get('x-object-meta-taste'))
+
+ destination = "/%s/%s" % (self.containername, self.objectname_2)
+ self.conn.copy_object(self.containername,
+ self.objectname,
+ destination,
+ headers={'x-object-meta-taste': 'Second'})
+ headers, data = self.conn.get_object(self.containername,
+ self.objectname_2)
+ self.assertEqual(self.test_data, data)
+ self.assertEqual('Something', headers.get('x-object-meta-color'))
+ self.assertEqual('Second', headers.get('x-object-meta-taste'))
+
+ destination = "/%s/%s" % (self.containername, self.objectname_2)
+ self.conn.copy_object(self.containername,
+ self.objectname,
+ destination,
+ headers={'x-object-meta-color': 'Else'},
+ fresh_metadata=True)
+
+ headers = self.conn.head_object(self.containername, self.objectname_2)
+ self.assertEqual('Else', headers.get('x-object-meta-color'))
+ self.assertIsNone(headers.get('x-object-meta-taste'))
+
def test_get_capabilities(self):
resp = self.conn.get_capabilities()
self.assertTrue(resp.get('swift'))
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py
index cce8c7a..5b04c44 100644
--- a/tests/unit/test_service.py
+++ b/tests/unit/test_service.py
@@ -69,6 +69,42 @@ class TestSwiftPostObject(unittest.TestCase):
self.assertRaises(SwiftError, self.spo, 1)
+class TestSwiftCopyObject(unittest.TestCase):
+
+ def setUp(self):
+ super(TestSwiftCopyObject, self).setUp()
+ self.sco = swiftclient.service.SwiftCopyObject
+
+ def test_create(self):
+ sco = self.sco('obj_name')
+
+ self.assertEqual(sco.object_name, 'obj_name')
+ self.assertIsNone(sco.destination)
+ self.assertFalse(sco.fresh_metadata)
+
+ sco = self.sco('obj_name',
+ {'destination': '/dest', 'fresh_metadata': True})
+
+ self.assertEqual(sco.object_name, 'obj_name')
+ self.assertEqual(sco.destination, '/dest/obj_name')
+ self.assertTrue(sco.fresh_metadata)
+
+ sco = self.sco('obj_name',
+ {'destination': '/dest/new_obj/a',
+ 'fresh_metadata': False})
+
+ self.assertEqual(sco.object_name, 'obj_name')
+ self.assertEqual(sco.destination, '/dest/new_obj/a')
+ self.assertFalse(sco.fresh_metadata)
+
+ def test_create_with_invalid_name(self):
+ # empty strings are not allowed as names
+ self.assertRaises(SwiftError, self.sco, '')
+
+ # names cannot be anything but strings
+ self.assertRaises(SwiftError, self.sco, 1)
+
+
class TestSwiftReader(unittest.TestCase):
def setUp(self):
@@ -2344,3 +2380,73 @@ class TestServicePost(_TestServiceBase):
res_iter.assert_called_with(
[tm_instance.object_uu_pool.submit()] * len(calls))
+
+
+class TestServiceCopy(_TestServiceBase):
+
+ def setUp(self):
+ super(TestServiceCopy, self).setUp()
+ self.opts = swiftclient.service._default_local_options.copy()
+
+ @mock.patch('swiftclient.service.MultiThreadingManager')
+ @mock.patch('swiftclient.service.interruptable_as_completed')
+ def test_object_copy(self, inter_compl, thread_manager):
+ """
+ Check copy method translates strings and objects to _copy_object_job
+ calls correctly
+ """
+ tm_instance = Mock()
+ thread_manager.return_value = tm_instance
+
+ self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]})
+ sco = swiftclient.service.SwiftCopyObject(
+ "test_sco",
+ options={'meta': ["meta1:test2"], "header": ["hdr1:test2"],
+ 'destination': "/cont_new/test_sco"})
+
+ res = SwiftService().copy('test_c', ['test_o', sco], self.opts)
+ res = list(res)
+
+ calls = [
+ mock.call(
+ SwiftService._create_container_job, 'cont_new', headers={}),
+ ]
+ tm_instance.container_pool.submit.assert_has_calls(calls,
+ any_order=True)
+ self.assertEqual(
+ tm_instance.container_pool.submit.call_count, len(calls))
+
+ calls = [
+ mock.call(
+ SwiftService._copy_object_job, 'test_c', 'test_o',
+ None,
+ {
+ "X-Object-Meta-Meta1": "test1",
+ "Hdr1": "test1"},
+ False),
+ mock.call(
+ SwiftService._copy_object_job, 'test_c', 'test_sco',
+ '/cont_new/test_sco',
+ {
+ "X-Object-Meta-Meta1": "test2",
+ "Hdr1": "test2"},
+ False),
+ ]
+ tm_instance.object_uu_pool.submit.assert_has_calls(calls)
+ self.assertEqual(
+ tm_instance.object_uu_pool.submit.call_count, len(calls))
+
+ inter_compl.assert_called_with(
+ [tm_instance.object_uu_pool.submit()] * len(calls))
+
+ def test_object_copy_fail_dest(self):
+ """
+ Destination in incorrect format and destination with object
+ used when multiple objects are copied raises SwiftError
+ """
+ with self.assertRaises(SwiftError):
+ list(SwiftService().copy('test_c', ['test_o'],
+ {'destination': 'cont'}))
+ with self.assertRaises(SwiftError):
+ list(SwiftService().copy('test_c', ['test_o', 'test_o2'],
+ {'destination': '/cont/obj'}))
diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py
index d82def6..407076b 100644
--- a/tests/unit/test_shell.py
+++ b/tests/unit/test_shell.py
@@ -1244,6 +1244,139 @@ class TestShell(unittest.TestCase):
self.assertTrue(output.err != '')
self.assertTrue(output.err.startswith('Usage'))
+ @mock.patch('swiftclient.service.Connection')
+ def test_copy_object_no_destination(self, connection):
+ argv = ["", "copy", "container", "object",
+ "--meta", "Color:Blue",
+ "--header", "content-type:text/plain"
+ ]
+ with CaptureOutput() as output:
+ swiftclient.shell.main(argv)
+ connection.return_value.copy_object.assert_called_with(
+ 'container', 'object', destination=None, fresh_metadata=False,
+ headers={
+ 'Content-Type': 'text/plain',
+ 'X-Object-Meta-Color': 'Blue'}, response_dict={})
+ self.assertEqual(output.out, 'container/object copied to <self>\n')
+
+ @mock.patch('swiftclient.service.Connection')
+ def test_copy_object(self, connection):
+ argv = ["", "copy", "container", "object",
+ "--meta", "Color:Blue",
+ "--header", "content-type:text/plain",
+ "--destination", "/c/o"
+ ]
+ with CaptureOutput() as output:
+ swiftclient.shell.main(argv)
+ connection.return_value.copy_object.assert_called_with(
+ 'container', 'object', destination="/c/o",
+ fresh_metadata=False,
+ headers={
+ 'Content-Type': 'text/plain',
+ 'X-Object-Meta-Color': 'Blue'}, response_dict={})
+ self.assertEqual(
+ output.out,
+ 'created container c\ncontainer/object copied to /c/o\n'
+ )
+
+ @mock.patch('swiftclient.service.Connection')
+ def test_copy_object_fresh_metadata(self, connection):
+ argv = ["", "copy", "container", "object",
+ "--meta", "Color:Blue", "--fresh-metadata",
+ "--header", "content-type:text/plain",
+ "--destination", "/c/o"
+ ]
+ swiftclient.shell.main(argv)
+ connection.return_value.copy_object.assert_called_with(
+ 'container', 'object', destination="/c/o", fresh_metadata=True,
+ headers={
+ 'Content-Type': 'text/plain',
+ 'X-Object-Meta-Color': 'Blue'}, response_dict={})
+
+ @mock.patch('swiftclient.service.Connection')
+ def test_copy_two_objects(self, connection):
+ argv = ["", "copy", "container", "object", "object2",
+ "--meta", "Color:Blue"]
+ connection.return_value.copy_object.return_value = None
+ swiftclient.shell.main(argv)
+ calls = [
+ mock.call(
+ 'container', 'object', destination=None,
+ fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'},
+ response_dict={}),
+ mock.call(
+ 'container', 'object2', destination=None,
+ fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'},
+ response_dict={})
+ ]
+ for call in calls:
+ self.assertIn(call, connection.return_value.copy_object.mock_calls)
+ self.assertEqual(len(connection.return_value.copy_object.mock_calls),
+ len(calls))
+
+ @mock.patch('swiftclient.service.Connection')
+ def test_copy_two_objects_destination(self, connection):
+ argv = ["", "copy", "container", "object", "object2",
+ "--meta", "Color:Blue", "--destination", "/c"]
+ swiftclient.shell.main(argv)
+ calls = [
+ mock.call(
+ 'container', 'object', destination="/c/object",
+ fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'},
+ response_dict={}),
+ mock.call(
+ 'container', 'object2', destination="/c/object2",
+ fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'},
+ response_dict={})
+ ]
+ connection.return_value.copy_object.assert_has_calls(calls)
+
+ @mock.patch('swiftclient.service.Connection')
+ def test_copy_two_objects_bad_destination(self, connection):
+ argv = ["", "copy", "container", "object", "object2",
+ "--meta", "Color:Blue", "--destination", "/c/o"]
+
+ with CaptureOutput() as output:
+ with self.assertRaises(SystemExit):
+ swiftclient.shell.main(argv)
+
+ self.assertEqual(
+ output.err,
+ 'Combination of multiple objects and destination '
+ 'including object is invalid\n')
+
+ @mock.patch('swiftclient.service.Connection')
+ def test_copy_object_bad_auth(self, connection):
+ argv = ["", "copy", "container", "object"]
+ connection.return_value.copy_object.side_effect = \
+ swiftclient.ClientException("bad auth")
+
+ with CaptureOutput() as output:
+ with self.assertRaises(SystemExit):
+ swiftclient.shell.main(argv)
+
+ self.assertEqual(output.err, 'bad auth\n')
+
+ def test_copy_object_not_enough_args(self):
+ argv = ["", "copy", "container"]
+
+ with CaptureOutput() as output:
+ with self.assertRaises(SystemExit):
+ swiftclient.shell.main(argv)
+
+ self.assertTrue(output.err != '')
+ self.assertTrue(output.err.startswith('Usage'))
+
+ def test_copy_bad_container(self):
+ argv = ["", "copy", "cont/ainer", "object"]
+
+ with CaptureOutput() as output:
+ with self.assertRaises(SystemExit):
+ swiftclient.shell.main(argv)
+
+ self.assertTrue(output.err != '')
+ self.assertTrue(output.err.startswith('WARN'))
+
@mock.patch('swiftclient.shell.generate_temp_url', return_value='')
def test_temp_url(self, temp_url):
argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o",
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py
index cbb95db..cc6a4ac 100644
--- a/tests/unit/test_swiftclient.py
+++ b/tests/unit/test_swiftclient.py
@@ -1398,6 +1398,109 @@ class TestPostObject(MockHttpTest):
])
+class TestCopyObject(MockHttpTest):
+
+ def test_server_error(self):
+ c.http_connection = self.fake_http_connection(500)
+ self.assertRaises(
+ c.ClientException, c.copy_object,
+ 'http://www.test.com/v1/AUTH', 'asdf', 'asdf', 'asdf')
+
+ def test_ok(self):
+ c.http_connection = self.fake_http_connection(200)
+ c.copy_object(
+ 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj',
+ destination='/container2/obj')
+ self.assertRequests([
+ ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', {
+ 'X-Auth-Token': 'token',
+ 'Destination': '/container2/obj',
+ }),
+ ])
+
+ def test_service_token(self):
+ c.http_connection = self.fake_http_connection(200)
+ c.copy_object('http://www.test.com/v1/AUTH', None, 'container',
+ 'obj', destination='/container2/obj',
+ service_token="TOKEN")
+ self.assertRequests([
+ ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', {
+ 'X-Service-Token': 'TOKEN',
+ 'Destination': '/container2/obj',
+
+ }),
+ ])
+
+ def test_headers(self):
+ c.http_connection = self.fake_http_connection(200)
+ c.copy_object(
+ 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj',
+ destination='/container2/obj',
+ headers={'some-hdr': 'a', 'other-hdr': 'b'})
+ self.assertRequests([
+ ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', {
+ 'X-Auth-Token': 'token',
+ 'Destination': '/container2/obj',
+ 'some-hdr': 'a',
+ 'other-hdr': 'b',
+ }),
+ ])
+
+ def test_fresh_metadata_default(self):
+ c.http_connection = self.fake_http_connection(200)
+ c.copy_object(
+ 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj',
+ '/container2/obj', {'x-fresh-metadata': 'hdr-value'})
+ self.assertRequests([
+ ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', {
+ 'X-Auth-Token': 'token',
+ 'Destination': '/container2/obj',
+ 'X-Fresh-Metadata': 'hdr-value',
+ }),
+ ])
+
+ def test_fresh_metadata_true(self):
+ c.http_connection = self.fake_http_connection(200)
+ c.copy_object(
+ 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj',
+ destination='/container2/obj',
+ headers={'x-fresh-metadata': 'hdr-value'},
+ fresh_metadata=True)
+ self.assertRequests([
+ ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', {
+ 'X-Auth-Token': 'token',
+ 'Destination': '/container2/obj',
+ 'X-Fresh-Metadata': 'true',
+ }),
+ ])
+
+ def test_fresh_metadata_false(self):
+ c.http_connection = self.fake_http_connection(200)
+ c.copy_object(
+ 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj',
+ destination='/container2/obj',
+ headers={'x-fresh-metadata': 'hdr-value'},
+ fresh_metadata=False)
+ self.assertRequests([
+ ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', {
+ 'x-auth-token': 'token',
+ 'Destination': '/container2/obj',
+ 'X-Fresh-Metadata': 'false',
+ }),
+ ])
+
+ def test_no_destination(self):
+ c.http_connection = self.fake_http_connection(200)
+ c.copy_object(
+ 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj')
+ self.assertRequests([
+ ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', {
+ 'x-auth-token': 'token',
+ 'Destination': '/container/obj',
+ }),
+ ])
+
+
class TestDeleteObject(MockHttpTest):
def test_ok(self):
@@ -2246,6 +2349,7 @@ class TestResponseDict(MockHttpTest):
('delete_container', 'c'),
('post_object', 'c', 'o', {}),
('put_object', 'c', 'o', 'body'),
+ ('copy_object', 'c', 'o'),
('delete_object', 'c', 'o')]
def fake_get_auth(*args, **kwargs):