diff options
Diffstat (limited to 'swiftclient')
| -rw-r--r-- | swiftclient/client.py | 105 | ||||
| -rw-r--r-- | swiftclient/service.py | 232 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 127 | ||||
| -rw-r--r-- | swiftclient/utils.py | 45 |
4 files changed, 470 insertions, 39 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py index f556afd..988c7d9 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -827,7 +827,8 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, def get_container(url, token, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, http_conn=None, - full_listing=False, service_token=None, headers=None): + full_listing=False, service_token=None, headers=None, + query_string=None): """ Get a listing of objects for the container. @@ -846,6 +847,7 @@ def get_container(url, token, container, marker=None, limit=None, of 10000 listings :param service_token: service auth token :param headers: additional headers to include in the request + :param query_string: if set will be appended with '?' to generated path :returns: a tuple of (response headers, a list of objects) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -889,6 +891,8 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&end_marker=%s' % quote(end_marker) if path: qs += '&path=%s' % quote(path) + if query_string: + qs += '&%s' % query_string.lstrip('?') if service_token: headers['X-Service-Token'] = service_token method = 'GET' @@ -950,7 +954,7 @@ def head_container(url, token, container, http_conn=None, headers=None, def put_container(url, token, container, headers=None, http_conn=None, - response_dict=None, service_token=None): + response_dict=None, service_token=None, query_string=None): """ Create a container @@ -963,6 +967,7 @@ def put_container(url, token, container, headers=None, http_conn=None, :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token + :param query_string: if set will be appended with '?' to generated path :raises ClientException: HTTP PUT request failed """ if http_conn: @@ -978,6 +983,8 @@ def put_container(url, token, container, headers=None, http_conn=None, headers['X-Service-Token'] = service_token if 'content-length' not in (k.lower() for k in headers): headers['Content-Length'] = '0' + if query_string: + path += '?' + query_string.lstrip('?') conn.request(method, path, '', headers) resp = conn.getresponse() body = resp.read() @@ -1031,7 +1038,8 @@ def post_container(url, token, container, headers, http_conn=None, def delete_container(url, token, container, http_conn=None, - response_dict=None, service_token=None): + response_dict=None, service_token=None, + query_string=None): """ Delete a container @@ -1043,6 +1051,7 @@ def delete_container(url, token, container, http_conn=None, :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token + :param query_string: if set will be appended with '?' to generated path :raises ClientException: HTTP DELETE request failed """ if http_conn: @@ -1053,6 +1062,8 @@ def delete_container(url, token, container, http_conn=None, headers = {'X-Auth-Token': token} if service_token: headers['X-Service-Token'] = service_token + if query_string: + path += '?' + query_string.lstrip('?') method = 'DELETE' conn.request(method, path, '', headers) resp = conn.getresponse() @@ -1319,6 +1330,70 @@ def post_object(url, token, container, name, headers, http_conn=None, raise ClientException.from_response(resp, 'Object POST failed', body) +def copy_object(url, token, container, name, destination=None, + headers=None, fresh_metadata=None, http_conn=None, + response_dict=None, service_token=None): + """ + Copy object + + :param url: storage URL + :param token: auth token; if None, no token will be sent + :param container: container name that the source object is in + :param name: source object name + :param destination: The container and object name of the destination object + in the form of /container/object; if None, the copy + will use the source as the destination. + :param headers: additional headers to include in the request + :param fresh_metadata: Enables object creation that omits existing user + metadata, default None + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param response_dict: an optional dictionary into which to place + the response - status, reason and headers + :param service_token: service auth token + :raises ClientException: HTTP COPY request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + + path = parsed.path + container = quote(container) + name = quote(name) + path = '%s/%s/%s' % (path.rstrip('/'), container, name) + + headers = dict(headers) if headers else {} + + if destination is not None: + headers['Destination'] = quote(destination) + elif container and name: + headers['Destination'] = '/%s/%s' % (container, name) + + if token is not None: + headers['X-Auth-Token'] = token + if service_token is not None: + headers['X-Service-Token'] = service_token + + if fresh_metadata is not None: + # remove potential fresh metadata headers + for fresh_hdr in [hdr for hdr in headers.keys() + if hdr.lower() == 'x-fresh-metadata']: + headers.pop(fresh_hdr) + headers['X-Fresh-Metadata'] = 'true' if fresh_metadata else 'false' + + conn.request('COPY', path, '', headers) + resp = conn.getresponse() + body = resp.read() + http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'COPY',), + {'headers': headers}, resp, body) + + store_response(resp, response_dict) + + if resp.status < 200 or resp.status >= 300: + raise ClientException.from_response(resp, 'Object COPY failed', body) + + def delete_object(url, token=None, container=None, name=None, http_conn=None, headers=None, proxy=None, query_string=None, response_dict=None, service_token=None): @@ -1625,7 +1700,7 @@ class Connection(object): def get_container(self, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, - full_listing=False, headers=None): + full_listing=False, headers=None, query_string=None): """Wrapper for :func:`get_container`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just @@ -1633,22 +1708,27 @@ class Connection(object): return self._retry(None, get_container, container, marker=marker, limit=limit, prefix=prefix, delimiter=delimiter, end_marker=end_marker, path=path, - full_listing=full_listing, headers=headers) + full_listing=full_listing, headers=headers, + query_string=query_string) - def put_container(self, container, headers=None, response_dict=None): + def put_container(self, container, headers=None, response_dict=None, + query_string=None): """Wrapper for :func:`put_container`""" return self._retry(None, put_container, container, headers=headers, - response_dict=response_dict) + response_dict=response_dict, + query_string=query_string) def post_container(self, container, headers, response_dict=None): """Wrapper for :func:`post_container`""" return self._retry(None, post_container, container, headers, response_dict=response_dict) - def delete_container(self, container, response_dict=None): + def delete_container(self, container, response_dict=None, + query_string=None): """Wrapper for :func:`delete_container`""" return self._retry(None, delete_container, container, - response_dict=response_dict) + response_dict=response_dict, + query_string=query_string) def head_object(self, container, obj, headers=None): """Wrapper for :func:`head_object`""" @@ -1711,6 +1791,13 @@ class Connection(object): return self._retry(None, post_object, container, obj, headers, response_dict=response_dict) + def copy_object(self, container, obj, destination=None, headers=None, + fresh_metadata=None, response_dict=None): + """Wrapper for :func:`copy_object`""" + return self._retry(None, copy_object, container, obj, destination, + headers, fresh_metadata, + response_dict=response_dict) + def delete_object(self, container, obj, query_string=None, response_dict=None): """Wrapper for :func:`delete_object`""" diff --git a/swiftclient/service.py b/swiftclient/service.py index 6ccba55..f204895 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -200,7 +200,9 @@ _default_local_options = { 'human': False, 'dir_marker': False, 'checksum': True, - 'shuffle': False + 'shuffle': False, + 'destination': None, + 'fresh_metadata': False, } POLICY = 'X-Storage-Policy' @@ -280,7 +282,7 @@ def split_headers(options, prefix=''): for item in options: split_item = item.split(':', 1) if len(split_item) == 2: - headers[(prefix + split_item[0]).title()] = split_item[1] + headers[(prefix + split_item[0]).title()] = split_item[1].strip() else: raise SwiftError( "Metadata parameter %s must contain a ':'.\n%s" @@ -321,13 +323,48 @@ class SwiftPostObject(object): specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not isinstance(object_name, string_types) or not object_name: + if not (isinstance(object_name, string_types) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) + self.object_name = object_name + self.options = options + + +class SwiftCopyObject(object): + """ + Class for specifying an object copy, + allowing the destination/headers/metadata/fresh_metadata to be specified + separately for each individual object. + destination and fresh_metadata should be set in options + """ + def __init__(self, object_name, options=None): + if not (isinstance(object_name, string_types) and object_name): + raise SwiftError( + "Object names must be specified as non-empty strings" + ) + + self.object_name = object_name + self.options = options + + if self.options is None: + self.destination = None + self.fresh_metadata = False else: - self.object_name = object_name - self.options = options + self.destination = self.options.get('destination') + self.fresh_metadata = self.options.get('fresh_metadata', False) + + if self.destination is not None: + destination_components = self.destination.split('/') + if destination_components[0] or len(destination_components) < 2: + raise SwiftError("destination must be in format /cont[/obj]") + if not destination_components[-1]: + raise SwiftError("destination must not end in a slash") + if len(destination_components) == 2: + # only container set in destination + self.destination = "{0}/{1}".format( + self.destination, object_name + ) class _SwiftReader(object): @@ -2391,6 +2428,191 @@ class SwiftService(object): return res + # Copy related methods + # + def copy(self, container, objects, options=None): + """ + Copy operations on a list of objects in a container. Destination + containers will be created. + + :param container: The container from which to copy the objects. + :param objects: A list of object names (strings) or SwiftCopyObject + instances containing an object name and an + options dict (can be None) to override the options for + that individual copy operation:: + + [ + 'object_name', + SwiftCopyObject( + 'object_name', + options={ + 'destination': '/container/object', + 'fresh_metadata': False, + ... + }), + ... + ] + + The options dict is described below. + :param options: A dictionary containing options to override the global + options specified during the service object creation. + These options are applied to all copy operations + performed by this call, unless overridden on a per + object basis. + The options "destination" and "fresh_metadata" do + not need to be set, in this case objects will be + copied onto themselves and metadata will not be + refreshed. + The option "destination" can also be specified in the + format '/container', in which case objects without an + explicit destination will be copied to the destination + /container/original_object_name. Combinations of + multiple objects and a destination in the format + '/container/object' is invalid. Possible options are + given below:: + + { + 'meta': [], + 'header': [], + 'destination': '/container/object', + 'fresh_metadata': False, + } + + :returns: A generator returning the results of copying the given list + of objects. + + :raises: SwiftError + """ + if options is not None: + options = dict(self._options, **options) + else: + options = self._options + + # Try to create the container, just in case it doesn't exist. If this + # fails, it might just be because the user doesn't have container PUT + # permissions, so we'll ignore any error. If there's really a problem, + # it'll surface on the first object COPY. + containers = set( + next(p for p in obj.destination.split("/") if p) + for obj in objects + if isinstance(obj, SwiftCopyObject) and obj.destination + ) + if options.get('destination'): + destination_split = options['destination'].split('/') + if destination_split[0]: + raise SwiftError("destination must be in format /cont[/obj]") + _str_objs = [ + o for o in objects if not isinstance(o, SwiftCopyObject) + ] + if len(destination_split) > 2 and len(_str_objs) > 1: + # TODO (clayg): could be useful to copy multiple objects into + # a destination like "/container/common/prefix/for/objects/" + # where the trailing "/" indicates the destination option is a + # prefix! + raise SwiftError("Combination of multiple objects and " + "destination including object is invalid") + if destination_split[-1] == '': + # N.B. this protects the above case + raise SwiftError("destination can not end in a slash") + containers.add(destination_split[1]) + + policy_header = {} + _header = split_headers(options["header"]) + if POLICY in _header: + policy_header[POLICY] = _header[POLICY] + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, cont, headers=policy_header) + for cont in containers + ] + + # wait for container creation jobs to complete before any COPY + for r in interruptable_as_completed(create_containers): + res = r.result() + yield res + + copy_futures = [] + copy_objects = self._make_copy_objects(objects, options) + for copy_object in copy_objects: + obj = copy_object.object_name + obj_options = copy_object.options + destination = copy_object.destination + fresh_metadata = copy_object.fresh_metadata + headers = split_headers( + options['meta'], 'X-Object-Meta-') + # add header options to the headers object for the request. + headers.update( + split_headers(options['header'], '')) + if obj_options is not None: + if 'meta' in obj_options: + headers.update( + split_headers( + obj_options['meta'], 'X-Object-Meta-' + ) + ) + if 'header' in obj_options: + headers.update( + split_headers(obj_options['header'], '') + ) + + copy = self.thread_manager.object_uu_pool.submit( + self._copy_object_job, container, obj, destination, + headers, fresh_metadata + ) + copy_futures.append(copy) + + for r in interruptable_as_completed(copy_futures): + res = r.result() + yield res + + @staticmethod + def _make_copy_objects(objects, options): + copy_objects = [] + + for o in objects: + if isinstance(o, string_types): + obj = SwiftCopyObject(o, options) + copy_objects.append(obj) + elif isinstance(o, SwiftCopyObject): + copy_objects.append(o) + else: + raise SwiftError( + "The copy operation takes only strings or " + "SwiftCopyObjects as input", + obj=o) + + return copy_objects + + @staticmethod + def _copy_object_job(conn, container, obj, destination, headers, + fresh_metadata): + response_dict = {} + res = { + 'success': True, + 'action': 'copy_object', + 'container': container, + 'object': obj, + 'destination': destination, + 'headers': headers, + 'fresh_metadata': fresh_metadata, + 'response_dict': response_dict + } + try: + conn.copy_object( + container, obj, destination=destination, headers=headers, + fresh_metadata=fresh_metadata, response_dict=response_dict) + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + res.update({ + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) + + return res + # Capabilities related methods # def capabilities(self, url=None, refresh_cache=False): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index ef165d8..be1888d 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,6 +17,7 @@ from __future__ import print_function, unicode_literals import argparse +import json import logging import signal import socket @@ -46,7 +47,7 @@ except ImportError: from pipes import quote as sh_quote BASENAME = 'swift' -commands = ('delete', 'download', 'list', 'post', 'stat', 'upload', +commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload', 'capabilities', 'info', 'tempurl', 'auth') @@ -248,7 +249,7 @@ Optional arguments: -H, --header <header:value> Adds a customized request header to the query, like "Range" or "If-Match". This option may be repeated. - Example --header "content-type:text/plain" + Example: --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. --ignore-checksum Turn off checksum validation for downloads. @@ -750,6 +751,105 @@ def st_post(parser, args, output_manager): output_manager.error(e.value) +st_copy_options = '''[--destination </container/object>] [--fresh-metadata] + [--meta <name:value>] [--header <header>] container object +''' + +st_copy_help = ''' +Copies object to new destination, optionally updates objects metadata. +If destination is not set, will update metadata of object + +Positional arguments: + container Name of container to copy from. + object Name of object to copy. Specify multiple times + for multiple objects + +Optional arguments: + -d, --destination </container[/object]> + The container and name of the destination object. Name + of destination object can be ommited, then will be + same as name of source object. Supplying multiple + objects and destination with object name is invalid. + -M, --fresh-metadata Copy the object without any existing metadata, + If not set, metadata will be preserved or appended + -m, --meta <name:value> + Sets a meta data item. This option may be repeated. + Example: -m Color:Blue -m Size:Large + -H, --header <header:value> + Adds a customized request header. + This option may be repeated. Example + -H "content-type:text/plain" -H "Content-Length: 4000" +'''.strip('\n') + + +def st_copy(parser, args, output_manager): + parser.add_argument( + '-d', '--destination', help='The container and name of the ' + 'destination object') + parser.add_argument( + '-M', '--fresh-metadata', action='store_true', + help='Copy the object without any existing metadata', default=False) + parser.add_argument( + '-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item. This option may be repeated. ' + 'Example: -m Color:Blue -m Size:Large') + parser.add_argument( + '-H', '--header', action='append', dest='header', + default=[], help='Adds a customized request header. ' + 'This option may be repeated. ' + 'Example: -H "content-type:text/plain" ' + '-H "Content-Length: 4000"') + (options, args) = parse_args(parser, args) + args = args[1:] + + with SwiftService(options=options) as swift: + try: + if len(args) >= 2: + container = args[0] + if '/' in container: + output_manager.error( + 'WARNING: / in container name; you might have ' + "meant '%s' instead of '%s'." % + (args[0].replace('/', ' ', 1), args[0])) + return + objects = [arg for arg in args[1:]] + + for r in swift.copy( + container=container, objects=objects, + options=options): + if r['success']: + if options['verbose']: + if r['action'] == 'copy_object': + output_manager.print_msg( + '%s/%s copied to %s' % ( + r['container'], + r['object'], + r['destination'] or '<self>')) + if r['action'] == 'create_container': + output_manager.print_msg( + 'created container %s' % r['container'] + ) + else: + error = r['error'] + if 'action' in r and r['action'] == 'create_container': + # it is not an error to be unable to create the + # container so print a warning and carry on + output_manager.warning( + 'Warning: failed to create container ' + "'%s': %s", container, error + ) + else: + output_manager.error("%s" % error) + else: + output_manager.error( + 'Usage: %s copy %s\n%s', BASENAME, + st_copy_options, st_copy_help) + return + + except SwiftError as e: + output_manager.error(e.value) + + st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>] [--segment-container <container>] [--leave-segments] [--object-threads <thread>] [--segment-threads <threads>] @@ -789,7 +889,7 @@ Optional arguments: Default is 10. -H, --header <header:value> Adds a customized request header. This option may be - repeated. Example -H "content-type:text/plain" + repeated. Example: -H "content-type:text/plain" -H "Content-Length: 4000". --use-slo When used in conjunction with --segment-size it will create a Static Large Object instead of the default @@ -840,7 +940,7 @@ def st_upload(parser, args, output_manager): parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' - ' This option may be repeated. Example -H "content-type:text/plain" ' + ' This option may be repeated. Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') parser.add_argument( '--use-slo', action='store_true', default=False, @@ -995,13 +1095,16 @@ def st_upload(parser, args, output_manager): output_manager.error(e.value) -st_capabilities_options = "[<proxy_url>]" +st_capabilities_options = "[--json] [<proxy_url>]" st_info_options = st_capabilities_options st_capabilities_help = ''' Retrieve capability of the proxy. Optional positional arguments: <proxy_url> Proxy URL of the cluster to retrieve capabilities. + +Optional arguments: + --json Print the cluster capabilities in JSON format. '''.strip('\n') st_info_help = st_capabilities_help @@ -1017,6 +1120,8 @@ def st_capabilities(parser, args, output_manager): key=lambda x: x[0]): output_manager.print_msg(" %s: %s" % (key, value)) + parser.add_argument('--json', action='store_true', + help='print capability information in json') (options, args) = parse_args(parser, args) if args and len(args) > 2: output_manager.error('Usage: %s capabilities %s\n%s', @@ -1034,9 +1139,14 @@ def st_capabilities(parser, args, output_manager): capabilities_result = swift.capabilities() capabilities = capabilities_result['capabilities'] - _print_compo_cap('Core', {'swift': capabilities['swift']}) - del capabilities['swift'] - _print_compo_cap('Additional middleware', capabilities) + if options['json']: + output_manager.print_msg( + json.dumps(capabilities, sort_keys=True, indent=2)) + else: + capabilities = dict(capabilities) + _print_compo_cap('Core', {'swift': capabilities['swift']}) + del capabilities['swift'] + _print_compo_cap('Additional middleware', capabilities) except SwiftError as e: output_manager.error(e.value) @@ -1272,6 +1382,7 @@ Positional arguments: for a container. post Updates meta information for the account, container, or object; creates containers if not present. + copy Copies object, optionally adds meta stat Displays information for the account, container, or object. upload Uploads files or directories to the given container. diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 0abaed6..10687bf 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -78,15 +78,20 @@ def generate_temp_url(path, seconds, key, method, absolute=False): :raises: TypeError if seconds is not an integer :return: the path portion of a temporary URL """ - if seconds < 0: - raise ValueError('seconds must be a positive integer') try: - if not absolute: - expiration = int(time.time() + seconds) - else: - expiration = int(seconds) - except TypeError: + seconds = int(seconds) + except ValueError: raise TypeError('seconds must be an integer') + if seconds < 0: + raise ValueError('seconds must be a positive integer') + + if isinstance(path, six.binary_type): + try: + path_for_body = path.decode('utf-8') + except UnicodeDecodeError: + raise ValueError('path must be representable as UTF-8') + else: + path_for_body = path standard_methods = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE'] if method.upper() not in standard_methods: @@ -94,18 +99,24 @@ def generate_temp_url(path, seconds, key, method, absolute=False): logger.warning('Non default HTTP method %s for tempurl specified, ' 'possibly an error', method.upper()) - hmac_body = '\n'.join([method.upper(), str(expiration), path]) + if not absolute: + expiration = int(time.time() + seconds) + else: + expiration = seconds + hmac_body = u'\n'.join([method.upper(), str(expiration), path_for_body]) # Encode to UTF-8 for py3 compatibility - sig = hmac.new(key.encode(), - hmac_body.encode(), - hashlib.sha1).hexdigest() - - return ('{path}?temp_url_sig=' - '{sig}&temp_url_expires={exp}'.format( - path=path, - sig=sig, - exp=expiration)) + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest() + + temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( + path=path_for_body, sig=sig, exp=expiration) + # Have return type match path from caller + if isinstance(path, six.binary_type): + return temp_url.encode('utf-8') + else: + return temp_url def parse_api_response(headers, body): |