diff options
Diffstat (limited to 'swiftclient')
| -rw-r--r-- | swiftclient/client.py | 128 | ||||
| -rw-r--r-- | swiftclient/service.py | 240 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 205 | ||||
| -rw-r--r-- | swiftclient/utils.py | 72 |
4 files changed, 561 insertions, 84 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py index 744a876..ee5a838 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -211,6 +211,13 @@ def quote(value, safe='/'): def encode_utf8(value): + if type(value) in six.integer_types + (float, bool): + # As of requests 2.11.0, headers must be byte- or unicode-strings. + # Convert some known-good types as a convenience for developers. + # Note that we *don't* convert subclasses, as they may have overriddden + # __str__ or __repr__. + # See https://github.com/kennethreitz/requests/pull/3366 for more info + value = str(value) if isinstance(value, six.text_type): value = value.encode('utf8') return value @@ -732,7 +739,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, if end_marker: qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) - headers = {'X-Auth-Token': token} + headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'} if service_token: headers['X-Service-Token'] = service_token method = 'GET' @@ -827,7 +834,8 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, def get_container(url, token, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, http_conn=None, - full_listing=False, service_token=None, headers=None): + full_listing=False, service_token=None, headers=None, + query_string=None): """ Get a listing of objects for the container. @@ -846,6 +854,7 @@ def get_container(url, token, container, marker=None, limit=None, of 10000 listings :param service_token: service auth token :param headers: additional headers to include in the request + :param query_string: if set will be appended with '?' to generated path :returns: a tuple of (response headers, a list of objects) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -857,6 +866,7 @@ def get_container(url, token, container, marker=None, limit=None, else: headers = {} headers['X-Auth-Token'] = token + headers['Accept-Encoding'] = 'gzip' if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, @@ -889,6 +899,8 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&end_marker=%s' % quote(end_marker) if path: qs += '&path=%s' % quote(path) + if query_string: + qs += '&%s' % query_string.lstrip('?') if service_token: headers['X-Service-Token'] = service_token method = 'GET' @@ -950,7 +962,7 @@ def head_container(url, token, container, http_conn=None, headers=None, def put_container(url, token, container, headers=None, http_conn=None, - response_dict=None, service_token=None): + response_dict=None, service_token=None, query_string=None): """ Create a container @@ -963,6 +975,7 @@ def put_container(url, token, container, headers=None, http_conn=None, :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token + :param query_string: if set will be appended with '?' to generated path :raises ClientException: HTTP PUT request failed """ if http_conn: @@ -978,6 +991,8 @@ def put_container(url, token, container, headers=None, http_conn=None, headers['X-Service-Token'] = service_token if 'content-length' not in (k.lower() for k in headers): headers['Content-Length'] = '0' + if query_string: + path += '?' + query_string.lstrip('?') conn.request(method, path, '', headers) resp = conn.getresponse() body = resp.read() @@ -1031,7 +1046,8 @@ def post_container(url, token, container, headers, http_conn=None, def delete_container(url, token, container, http_conn=None, - response_dict=None, service_token=None): + response_dict=None, service_token=None, + query_string=None): """ Delete a container @@ -1043,6 +1059,7 @@ def delete_container(url, token, container, http_conn=None, :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token + :param query_string: if set will be appended with '?' to generated path :raises ClientException: HTTP DELETE request failed """ if http_conn: @@ -1053,6 +1070,8 @@ def delete_container(url, token, container, http_conn=None, headers = {'X-Auth-Token': token} if service_token: headers['X-Service-Token'] = service_token + if query_string: + path += '?' + query_string.lstrip('?') method = 'DELETE' conn.request(method, path, '', headers) resp = conn.getresponse() @@ -1319,6 +1338,70 @@ def post_object(url, token, container, name, headers, http_conn=None, raise ClientException.from_response(resp, 'Object POST failed', body) +def copy_object(url, token, container, name, destination=None, + headers=None, fresh_metadata=None, http_conn=None, + response_dict=None, service_token=None): + """ + Copy object + + :param url: storage URL + :param token: auth token; if None, no token will be sent + :param container: container name that the source object is in + :param name: source object name + :param destination: The container and object name of the destination object + in the form of /container/object; if None, the copy + will use the source as the destination. + :param headers: additional headers to include in the request + :param fresh_metadata: Enables object creation that omits existing user + metadata, default None + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param response_dict: an optional dictionary into which to place + the response - status, reason and headers + :param service_token: service auth token + :raises ClientException: HTTP COPY request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + + path = parsed.path + container = quote(container) + name = quote(name) + path = '%s/%s/%s' % (path.rstrip('/'), container, name) + + headers = dict(headers) if headers else {} + + if destination is not None: + headers['Destination'] = quote(destination) + elif container and name: + headers['Destination'] = '/%s/%s' % (container, name) + + if token is not None: + headers['X-Auth-Token'] = token + if service_token is not None: + headers['X-Service-Token'] = service_token + + if fresh_metadata is not None: + # remove potential fresh metadata headers + for fresh_hdr in [hdr for hdr in headers.keys() + if hdr.lower() == 'x-fresh-metadata']: + headers.pop(fresh_hdr) + headers['X-Fresh-Metadata'] = 'true' if fresh_metadata else 'false' + + conn.request('COPY', path, '', headers) + resp = conn.getresponse() + body = resp.read() + http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'COPY',), + {'headers': headers}, resp, body) + + store_response(resp, response_dict) + + if resp.status < 200 or resp.status >= 300: + raise ClientException.from_response(resp, 'Object COPY failed', body) + + def delete_object(url, token=None, container=None, name=None, http_conn=None, headers=None, proxy=None, query_string=None, response_dict=None, service_token=None): @@ -1382,10 +1465,11 @@ def get_capabilities(http_conn): :raises ClientException: HTTP Capabilities GET failed """ parsed, conn = http_conn - conn.request('GET', parsed.path, '') + headers = {'Accept-Encoding': 'gzip'} + conn.request('GET', parsed.path, '', headers) resp = conn.getresponse() body = resp.read() - http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body) + http_log((parsed.geturl(), 'GET',), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response( resp, 'Capabilities GET failed', body) @@ -1544,7 +1628,7 @@ class Connection(object): backoff = self.starting_backoff caller_response_dict = kwargs.pop('response_dict', None) self.attempts = kwargs.pop('attempts', 0) - while self.attempts <= self.retries: + while self.attempts <= self.retries or retried_auth: self.attempts += 1 try: if not self.url or not self.token: @@ -1573,9 +1657,6 @@ class Connection(object): self.http_conn = None except ClientException as err: self._add_response_dict(caller_response_dict, kwargs) - if self.attempts > self.retries or err.http_status is None: - logger.exception(err) - raise if err.http_status == 401: self.url = self.token = self.service_token = None if retried_auth or not all((self.authurl, @@ -1584,6 +1665,9 @@ class Connection(object): logger.exception(err) raise retried_auth = True + elif self.attempts > self.retries or err.http_status is None: + logger.exception(err) + raise elif err.http_status == 408: self.http_conn = None elif 500 <= err.http_status <= 599: @@ -1625,7 +1709,7 @@ class Connection(object): def get_container(self, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, - full_listing=False, headers=None): + full_listing=False, headers=None, query_string=None): """Wrapper for :func:`get_container`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just @@ -1633,22 +1717,27 @@ class Connection(object): return self._retry(None, get_container, container, marker=marker, limit=limit, prefix=prefix, delimiter=delimiter, end_marker=end_marker, path=path, - full_listing=full_listing, headers=headers) + full_listing=full_listing, headers=headers, + query_string=query_string) - def put_container(self, container, headers=None, response_dict=None): + def put_container(self, container, headers=None, response_dict=None, + query_string=None): """Wrapper for :func:`put_container`""" return self._retry(None, put_container, container, headers=headers, - response_dict=response_dict) + response_dict=response_dict, + query_string=query_string) def post_container(self, container, headers, response_dict=None): """Wrapper for :func:`post_container`""" return self._retry(None, post_container, container, headers, response_dict=response_dict) - def delete_container(self, container, response_dict=None): + def delete_container(self, container, response_dict=None, + query_string=None): """Wrapper for :func:`delete_container`""" return self._retry(None, delete_container, container, - response_dict=response_dict) + response_dict=response_dict, + query_string=query_string) def head_object(self, container, obj, headers=None): """Wrapper for :func:`head_object`""" @@ -1711,6 +1800,13 @@ class Connection(object): return self._retry(None, post_object, container, obj, headers, response_dict=response_dict) + def copy_object(self, container, obj, destination=None, headers=None, + fresh_metadata=None, response_dict=None): + """Wrapper for :func:`copy_object`""" + return self._retry(None, copy_object, container, obj, destination, + headers, fresh_metadata, + response_dict=response_dict) + def delete_object(self, container, obj, query_string=None, response_dict=None): """Wrapper for :func:`delete_object`""" diff --git a/swiftclient/service.py b/swiftclient/service.py index 12d3f21..f204895 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -200,7 +200,9 @@ _default_local_options = { 'human': False, 'dir_marker': False, 'checksum': True, - 'shuffle': False + 'shuffle': False, + 'destination': None, + 'fresh_metadata': False, } POLICY = 'X-Storage-Policy' @@ -280,7 +282,7 @@ def split_headers(options, prefix=''): for item in options: split_item = item.split(':', 1) if len(split_item) == 2: - headers[(prefix + split_item[0]).title()] = split_item[1] + headers[(prefix + split_item[0]).title()] = split_item[1].strip() else: raise SwiftError( "Metadata parameter %s must contain a ':'.\n%s" @@ -321,13 +323,48 @@ class SwiftPostObject(object): specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not isinstance(object_name, string_types) or not object_name: + if not (isinstance(object_name, string_types) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) + self.object_name = object_name + self.options = options + + +class SwiftCopyObject(object): + """ + Class for specifying an object copy, + allowing the destination/headers/metadata/fresh_metadata to be specified + separately for each individual object. + destination and fresh_metadata should be set in options + """ + def __init__(self, object_name, options=None): + if not (isinstance(object_name, string_types) and object_name): + raise SwiftError( + "Object names must be specified as non-empty strings" + ) + + self.object_name = object_name + self.options = options + + if self.options is None: + self.destination = None + self.fresh_metadata = False else: - self.object_name = object_name - self.options = options + self.destination = self.options.get('destination') + self.fresh_metadata = self.options.get('fresh_metadata', False) + + if self.destination is not None: + destination_components = self.destination.split('/') + if destination_components[0] or len(destination_components) < 2: + raise SwiftError("destination must be in format /cont[/obj]") + if not destination_components[-1]: + raise SwiftError("destination must not end in a slash") + if len(destination_components) == 2: + # only container set in destination + self.destination = "{0}/{1}".format( + self.destination, object_name + ) class _SwiftReader(object): @@ -336,7 +373,7 @@ class _SwiftReader(object): errors on failures caused by either invalid md5sum or size of the data read. """ - def __init__(self, path, body, headers): + def __init__(self, path, body, headers, checksum=True): self._path = path self._body = body self._actual_read = 0 @@ -345,7 +382,7 @@ class _SwiftReader(object): self._expected_etag = headers.get('etag') if ('x-object-manifest' not in headers - and 'x-static-large-object' not in headers): + and 'x-static-large-object' not in headers and checksum): self._actual_md5 = md5() if 'content-length' in headers: @@ -980,6 +1017,7 @@ class SwiftService(object): 'header': [], 'skip_identical': False, 'out_directory': None, + 'checksum': True, 'out_file': None, 'remove_prefix': False, 'shuffle' : False @@ -1135,7 +1173,8 @@ class SwiftService(object): headers_receipt = time() - obj_body = _SwiftReader(path, body, headers) + obj_body = _SwiftReader(path, body, headers, + options.get('checksum', True)) no_file = options['no_download'] if out_file == "-" and not no_file: @@ -2389,6 +2428,191 @@ class SwiftService(object): return res + # Copy related methods + # + def copy(self, container, objects, options=None): + """ + Copy operations on a list of objects in a container. Destination + containers will be created. + + :param container: The container from which to copy the objects. + :param objects: A list of object names (strings) or SwiftCopyObject + instances containing an object name and an + options dict (can be None) to override the options for + that individual copy operation:: + + [ + 'object_name', + SwiftCopyObject( + 'object_name', + options={ + 'destination': '/container/object', + 'fresh_metadata': False, + ... + }), + ... + ] + + The options dict is described below. + :param options: A dictionary containing options to override the global + options specified during the service object creation. + These options are applied to all copy operations + performed by this call, unless overridden on a per + object basis. + The options "destination" and "fresh_metadata" do + not need to be set, in this case objects will be + copied onto themselves and metadata will not be + refreshed. + The option "destination" can also be specified in the + format '/container', in which case objects without an + explicit destination will be copied to the destination + /container/original_object_name. Combinations of + multiple objects and a destination in the format + '/container/object' is invalid. Possible options are + given below:: + + { + 'meta': [], + 'header': [], + 'destination': '/container/object', + 'fresh_metadata': False, + } + + :returns: A generator returning the results of copying the given list + of objects. + + :raises: SwiftError + """ + if options is not None: + options = dict(self._options, **options) + else: + options = self._options + + # Try to create the container, just in case it doesn't exist. If this + # fails, it might just be because the user doesn't have container PUT + # permissions, so we'll ignore any error. If there's really a problem, + # it'll surface on the first object COPY. + containers = set( + next(p for p in obj.destination.split("/") if p) + for obj in objects + if isinstance(obj, SwiftCopyObject) and obj.destination + ) + if options.get('destination'): + destination_split = options['destination'].split('/') + if destination_split[0]: + raise SwiftError("destination must be in format /cont[/obj]") + _str_objs = [ + o for o in objects if not isinstance(o, SwiftCopyObject) + ] + if len(destination_split) > 2 and len(_str_objs) > 1: + # TODO (clayg): could be useful to copy multiple objects into + # a destination like "/container/common/prefix/for/objects/" + # where the trailing "/" indicates the destination option is a + # prefix! + raise SwiftError("Combination of multiple objects and " + "destination including object is invalid") + if destination_split[-1] == '': + # N.B. this protects the above case + raise SwiftError("destination can not end in a slash") + containers.add(destination_split[1]) + + policy_header = {} + _header = split_headers(options["header"]) + if POLICY in _header: + policy_header[POLICY] = _header[POLICY] + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, cont, headers=policy_header) + for cont in containers + ] + + # wait for container creation jobs to complete before any COPY + for r in interruptable_as_completed(create_containers): + res = r.result() + yield res + + copy_futures = [] + copy_objects = self._make_copy_objects(objects, options) + for copy_object in copy_objects: + obj = copy_object.object_name + obj_options = copy_object.options + destination = copy_object.destination + fresh_metadata = copy_object.fresh_metadata + headers = split_headers( + options['meta'], 'X-Object-Meta-') + # add header options to the headers object for the request. + headers.update( + split_headers(options['header'], '')) + if obj_options is not None: + if 'meta' in obj_options: + headers.update( + split_headers( + obj_options['meta'], 'X-Object-Meta-' + ) + ) + if 'header' in obj_options: + headers.update( + split_headers(obj_options['header'], '') + ) + + copy = self.thread_manager.object_uu_pool.submit( + self._copy_object_job, container, obj, destination, + headers, fresh_metadata + ) + copy_futures.append(copy) + + for r in interruptable_as_completed(copy_futures): + res = r.result() + yield res + + @staticmethod + def _make_copy_objects(objects, options): + copy_objects = [] + + for o in objects: + if isinstance(o, string_types): + obj = SwiftCopyObject(o, options) + copy_objects.append(obj) + elif isinstance(o, SwiftCopyObject): + copy_objects.append(o) + else: + raise SwiftError( + "The copy operation takes only strings or " + "SwiftCopyObjects as input", + obj=o) + + return copy_objects + + @staticmethod + def _copy_object_job(conn, container, obj, destination, headers, + fresh_metadata): + response_dict = {} + res = { + 'success': True, + 'action': 'copy_object', + 'container': container, + 'object': obj, + 'destination': destination, + 'headers': headers, + 'fresh_metadata': fresh_metadata, + 'response_dict': response_dict + } + try: + conn.copy_object( + container, obj, destination=destination, headers=headers, + fresh_metadata=fresh_metadata, response_dict=response_dict) + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + res.update({ + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) + + return res + # Capabilities related methods # def capabilities(self, url=None, refresh_cache=False): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 3e8de7d..3f8aad6 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,6 +17,7 @@ from __future__ import print_function, unicode_literals import argparse +import json import logging import signal import socket @@ -24,7 +25,7 @@ import socket from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join from six import text_type, PY2 -from six.moves.urllib.parse import unquote +from six.moves.urllib.parse import unquote, urlparse from sys import argv as sys_argv, exit, stderr from time import gmtime, strftime @@ -46,7 +47,7 @@ except ImportError: from pipes import quote as sh_quote BASENAME = 'swift' -commands = ('delete', 'download', 'list', 'post', 'stat', 'upload', +commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload', 'capabilities', 'info', 'tempurl', 'auth') @@ -57,7 +58,7 @@ def immediate_exit(signum, frame): st_delete_options = '''[--all] [--leave-segments] [--object-threads <threads>] [--container-threads <threads>] - [<container>] [<object>] [...] + [<container> [<object>] [...]] ''' st_delete_help = ''' @@ -204,22 +205,22 @@ def st_delete(parser, args, output_manager): st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>] [--output <out_file>] [--output-dir <out_directory>] - [--object-threads <threads>] + [--object-threads <threads>] [--ignore-checksum] [--container-threads <threads>] [--no-download] [--skip-identical] [--remove-prefix] [--header <header:value>] [--no-shuffle] - <container> <object> + [<container> [<object>] [...]] ''' st_download_help = ''' Download objects from containers. Positional arguments: - <container> Name of container to download from. To download a - whole account, omit this and specify --all. - <object> Name of object to download. Specify multiple times - for multiple objects. Omit this to download all - objects from the container. + [<container>] Name of container to download from. To download a + whole account, omit this and specify --all. + [<object>] Name of object to download. Specify multiple times + for multiple objects. Omit this to download all + objects from the container. Optional arguments: -a, --all Indicates that you really want to download @@ -248,9 +249,10 @@ Optional arguments: -H, --header <header:value> Adds a customized request header to the query, like "Range" or "If-Match". This option may be repeated. - Example --header "content-type:text/plain" + Example: --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. + --ignore-checksum Turn off checksum validation for downloads. --no-shuffle By default, when downloading a complete account or container, download order is randomised in order to reduce the load on individual drives when multiple @@ -309,6 +311,9 @@ def st_download(parser, args, output_manager): default=False, help='Skip downloading files that are identical on ' 'both sides.') parser.add_argument( + '--ignore-checksum', action='store_false', dest='checksum', + default=True, help='Turn off checksum validation for downloads.') + parser.add_argument( '--no-shuffle', action='store_false', dest='shuffle', default=True, help='By default, download order is randomised in order ' 'to reduce the load on individual drives when multiple clients are ' @@ -440,14 +445,14 @@ def st_download(parser, args, output_manager): st_list_options = '''[--long] [--lh] [--totals] [--prefix <prefix>] - [--delimiter <delimiter>] [container] + [--delimiter <delimiter>] [<container>] ''' st_list_help = ''' Lists the containers for the account or the objects for a container. Positional arguments: - [container] Name of container to list object in. + [<container>] Name of container to list object in. Optional arguments: -l, --long Long listing format, similar to ls -l. @@ -576,15 +581,15 @@ def st_list(parser, args, output_manager): st_stat_options = '''[--lh] - [container] [object] + [<container> [<object>]] ''' st_stat_help = ''' Displays information for the account, container, or object. Positional arguments: - [container] Name of container to stat from. - [object] Name of object to stat. + [<container>] Name of container to stat from. + [<object>] Name of object to stat. Optional arguments: --lh Report sizes in human readable format similar to @@ -650,7 +655,7 @@ def st_stat(parser, args, output_manager): st_post_options = '''[--read-acl <acl>] [--write-acl <acl>] [--sync-to] [--sync-key <sync-key>] [--meta <name:value>] [--header <header>] - [container] [object] + [<container> [<object>]] ''' st_post_help = ''' @@ -658,8 +663,8 @@ Updates meta information for the account, container, or object. If the container is not found, it will be created automatically. Positional arguments: - [container] Name of container to post to. - [object] Name of object to post. + [<container>] Name of container to post to. + [<object>] Name of object to post. Optional arguments: -r, --read-acl <acl> Read ACL for containers. Quick summary of ACL syntax: @@ -748,6 +753,106 @@ def st_post(parser, args, output_manager): output_manager.error(e.value) +st_copy_options = '''[--destination </container/object>] [--fresh-metadata] + [--meta <name:value>] [--header <header>] <container> + <object> [<object>] [...] +''' + +st_copy_help = ''' +Copies object to new destination, optionally updates objects metadata. +If destination is not set, will update metadata of object + +Positional arguments: + <container> Name of container to copy from. + <object> Name of object to copy. Specify multiple times + for multiple objects + +Optional arguments: + -d, --destination </container[/object]> + The container and name of the destination object. Name + of destination object can be ommited, then will be + same as name of source object. Supplying multiple + objects and destination with object name is invalid. + -M, --fresh-metadata Copy the object without any existing metadata, + If not set, metadata will be preserved or appended + -m, --meta <name:value> + Sets a meta data item. This option may be repeated. + Example: -m Color:Blue -m Size:Large + -H, --header <header:value> + Adds a customized request header. + This option may be repeated. Example + -H "content-type:text/plain" -H "Content-Length: 4000" +'''.strip('\n') + + +def st_copy(parser, args, output_manager): + parser.add_argument( + '-d', '--destination', help='The container and name of the ' + 'destination object') + parser.add_argument( + '-M', '--fresh-metadata', action='store_true', + help='Copy the object without any existing metadata', default=False) + parser.add_argument( + '-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item. This option may be repeated. ' + 'Example: -m Color:Blue -m Size:Large') + parser.add_argument( + '-H', '--header', action='append', dest='header', + default=[], help='Adds a customized request header. ' + 'This option may be repeated. ' + 'Example: -H "content-type:text/plain" ' + '-H "Content-Length: 4000"') + (options, args) = parse_args(parser, args) + args = args[1:] + + with SwiftService(options=options) as swift: + try: + if len(args) >= 2: + container = args[0] + if '/' in container: + output_manager.error( + 'WARNING: / in container name; you might have ' + "meant '%s' instead of '%s'." % + (args[0].replace('/', ' ', 1), args[0])) + return + objects = [arg for arg in args[1:]] + + for r in swift.copy( + container=container, objects=objects, + options=options): + if r['success']: + if options['verbose']: + if r['action'] == 'copy_object': + output_manager.print_msg( + '%s/%s copied to %s' % ( + r['container'], + r['object'], + r['destination'] or '<self>')) + if r['action'] == 'create_container': + output_manager.print_msg( + 'created container %s' % r['container'] + ) + else: + error = r['error'] + if 'action' in r and r['action'] == 'create_container': + # it is not an error to be unable to create the + # container so print a warning and carry on + output_manager.warning( + 'Warning: failed to create container ' + "'%s': %s", container, error + ) + else: + output_manager.error("%s" % error) + else: + output_manager.error( + 'Usage: %s copy %s\n%s', BASENAME, + st_copy_options, st_copy_help) + return + + except SwiftError as e: + output_manager.error(e.value) + + st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>] [--segment-container <container>] [--leave-segments] [--object-threads <thread>] [--segment-threads <threads>] @@ -787,7 +892,7 @@ Optional arguments: Default is 10. -H, --header <header:value> Adds a customized request header. This option may be - repeated. Example -H "content-type:text/plain" + repeated. Example: -H "content-type:text/plain" -H "Content-Length: 4000". --use-slo When used in conjunction with --segment-size it will create a Static Large Object instead of the default @@ -838,7 +943,7 @@ def st_upload(parser, args, output_manager): parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' - ' This option may be repeated. Example -H "content-type:text/plain" ' + ' This option may be repeated. Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') parser.add_argument( '--use-slo', action='store_true', default=False, @@ -993,13 +1098,16 @@ def st_upload(parser, args, output_manager): output_manager.error(e.value) -st_capabilities_options = "[<proxy_url>]" +st_capabilities_options = "[--json] [<proxy_url>]" st_info_options = st_capabilities_options st_capabilities_help = ''' Retrieve capability of the proxy. Optional positional arguments: <proxy_url> Proxy URL of the cluster to retrieve capabilities. + +Optional arguments: + --json Print the cluster capabilities in JSON format. '''.strip('\n') st_info_help = st_capabilities_help @@ -1015,6 +1123,8 @@ def st_capabilities(parser, args, output_manager): key=lambda x: x[0]): output_manager.print_msg(" %s: %s" % (key, value)) + parser.add_argument('--json', action='store_true', + help='print capability information in json') (options, args) = parse_args(parser, args) if args and len(args) > 2: output_manager.error('Usage: %s capabilities %s\n%s', @@ -1032,9 +1142,14 @@ def st_capabilities(parser, args, output_manager): capabilities_result = swift.capabilities() capabilities = capabilities_result['capabilities'] - _print_compo_cap('Core', {'swift': capabilities['swift']}) - del capabilities['swift'] - _print_compo_cap('Additional middleware', capabilities) + if options['json']: + output_manager.print_msg( + json.dumps(capabilities, sort_keys=True, indent=2)) + else: + capabilities = dict(capabilities) + _print_compo_cap('Core', {'swift': capabilities['swift']}) + del capabilities['swift'] + _print_compo_cap('Additional middleware', capabilities) except SwiftError as e: output_manager.error(e.value) @@ -1095,8 +1210,9 @@ Positional arguments: <seconds> The amount of time in seconds the temporary URL will be valid for; or, if --absolute is passed, the Unix timestamp when the temporary URL will expire. - <path> The full path to the Swift object. Example: - /v1/AUTH_account/c/o. + <path> The full path or storage URL to the Swift object. + Example: /v1/AUTH_account/c/o + or: http://saio:8080/v1/AUTH_account/c/o <key> The secret temporary URL key set on the Swift cluster. To set a key, run \'swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' @@ -1123,17 +1239,24 @@ def st_tempurl(parser, args, thread_manager): st_tempurl_options, st_tempurl_help) return method, seconds, path, key = args[:4] - try: - seconds = int(seconds) - except ValueError: - thread_manager.error('Seconds must be an integer') - return + + parsed = urlparse(path) + if method.upper() not in ['GET', 'PUT', 'HEAD', 'POST', 'DELETE']: thread_manager.print_msg('WARNING: Non default HTTP method %s for ' 'tempurl specified, possibly an error' % method.upper()) - url = generate_temp_url(path, seconds, key, method, - absolute=options['absolute_expiry']) + try: + path = generate_temp_url(parsed.path, seconds, key, method, + absolute=options['absolute_expiry']) + except ValueError as err: + thread_manager.error(err) + return + + if parsed.scheme and parsed.netloc: + url = "%s://%s%s" % (parsed.scheme, parsed.netloc, path) + else: + url = path thread_manager.print_msg(url) @@ -1173,16 +1296,16 @@ class HelpFormatter(argparse.HelpFormatter): def parse_args(parser, args, enforce_requires=True): options, args = parser.parse_known_args(args or ['-h']) options = vars(options) - if enforce_requires and (options['debug'] or options['info']): + if enforce_requires and (options.get('debug') or options.get('info')): logging.getLogger("swiftclient") - if options['debug']: + if options.get('debug'): logging.basicConfig(level=logging.DEBUG) logging.getLogger('iso8601').setLevel(logging.WARNING) client_logger_settings['redact_sensitive_headers'] = False - elif options['info']: + elif options.get('info'): logging.basicConfig(level=logging.INFO) - if args and options['help']: + if args and options.get('help'): _help = globals().get('st_%s_help' % args[0], "no help for %s" % args[0]) print(_help) @@ -1270,6 +1393,7 @@ Positional arguments: for a container. post Updates meta information for the account, container, or object; creates containers if not present. + copy Copies object, optionally adds meta stat Displays information for the account, container, or object. upload Uploads files or directories to the given container. @@ -1526,6 +1650,11 @@ Examples: with OutputManager() as output: parser.usage = globals()['st_%s_help' % args[0]] + if options['insecure']: + import requests + from requests.packages.urllib3.exceptions import \ + InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) try: globals()['st_%s' % args[0]](parser, argv[1:], output) except ClientException as err: diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 0abaed6..0d1104e 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous utility functions for use with Swift.""" +import gzip import hashlib import hmac import json @@ -67,26 +68,43 @@ def generate_temp_url(path, seconds, key, method, absolute=False): :param path: The full path to the Swift object. Example: /v1/AUTH_account/c/o. - :param seconds: The amount of time in seconds the temporary URL will - be valid for. + :param seconds: If absolute is False then this specifies the amount of time + in seconds for which the temporary URL will be valid. If absolute is + True then this specifies an absolute time at which the temporary URL + will expire. :param key: The secret temporary URL key set on the Swift cluster. To set a key, run 'swift post -m "Temp-URL-Key: <substitute tempurl key here>"' :param method: A HTTP method, typically either GET or PUT, to allow for this temporary URL. - :raises: ValueError if seconds is not a positive integer - :raises: TypeError if seconds is not an integer + :param absolute: if True then the seconds parameter is interpreted as an + absolute Unix time, otherwise seconds is interpreted as a relative time + offset from current time. + :raises: ValueError if seconds is not a whole number or path is not to + an object. :return: the path portion of a temporary URL """ - if seconds < 0: - raise ValueError('seconds must be a positive integer') try: - if not absolute: - expiration = int(time.time() + seconds) - else: - expiration = int(seconds) - except TypeError: - raise TypeError('seconds must be an integer') + seconds = float(seconds) + if not seconds.is_integer(): + raise ValueError() + seconds = int(seconds) + if seconds < 0: + raise ValueError() + except ValueError: + raise ValueError('seconds must be a whole number') + + if isinstance(path, six.binary_type): + try: + path_for_body = path.decode('utf-8') + except UnicodeDecodeError: + raise ValueError('path must be representable as UTF-8') + else: + path_for_body = path + + parts = path_for_body.split('/', 4) + if len(parts) != 5 or parts[0] or not all(parts[1:]): + raise ValueError('path must be full path to an object e.g. /v1/a/c/o') standard_methods = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE'] if method.upper() not in standard_methods: @@ -94,21 +112,31 @@ def generate_temp_url(path, seconds, key, method, absolute=False): logger.warning('Non default HTTP method %s for tempurl specified, ' 'possibly an error', method.upper()) - hmac_body = '\n'.join([method.upper(), str(expiration), path]) + if not absolute: + expiration = int(time.time() + seconds) + else: + expiration = seconds + hmac_body = u'\n'.join([method.upper(), str(expiration), path_for_body]) # Encode to UTF-8 for py3 compatibility - sig = hmac.new(key.encode(), - hmac_body.encode(), - hashlib.sha1).hexdigest() - - return ('{path}?temp_url_sig=' - '{sig}&temp_url_expires={exp}'.format( - path=path, - sig=sig, - exp=expiration)) + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest() + + temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( + path=path_for_body, sig=sig, exp=expiration) + # Have return type match path from caller + if isinstance(path, six.binary_type): + return temp_url.encode('utf-8') + else: + return temp_url def parse_api_response(headers, body): + if headers.get('content-encoding') == 'gzip': + with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz: + body = gz.read() + charset = 'utf-8' # Swift *should* be speaking UTF-8, but check content-type just in case content_type = headers.get('content-type', '') |