object versioning features

 * add --versions to list
 * add --versions to delete
 * add --version-id to stat
 * add --version-id to delete
 * add --version-id to download

Change-Id: I89802064921778fee7efe57c7d60c976cdde3a27

4 files changed, 163 insertions, 28 deletions

diff --git a/swiftclient/client.py b/swiftclient/client.py
index 448bb46..449b6cd 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -921,7 +921,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None,
 
 def get_container(url, token, container, marker=None, limit=None,
                   prefix=None, delimiter=None, end_marker=None,
-                  path=None, http_conn=None,
+                  version_marker=None, path=None, http_conn=None,
                   full_listing=False, service_token=None, headers=None,
                   query_string=None):
     """
@@ -935,6 +935,7 @@ def get_container(url, token, container, marker=None, limit=None,
     :param prefix: prefix query
     :param delimiter: string to delimit the queries on
     :param end_marker: marker query
+    :param version_marker: version marker query
     :param path: path query (equivalent: "delimiter=/" and "prefix=path/")
     :param http_conn: a tuple of (parsed url, HTTPConnection object),
                       (If None, it will create the conn object)
@@ -951,17 +952,20 @@ def get_container(url, token, container, marker=None, limit=None,
         http_conn = http_connection(url)
     if full_listing:
         rv = get_container(url, token, container, marker, limit, prefix,
-                           delimiter, end_marker, path, http_conn,
-                           service_token=service_token, headers=headers)
+                           delimiter, end_marker, version_marker, path=path,
+                           http_conn=http_conn, service_token=service_token,
+                           headers=headers)
         listing = rv[1]
         while listing:
             if not delimiter:
                 marker = listing[-1]['name']
             else:
                 marker = listing[-1].get('name', listing[-1].get('subdir'))
+            version_marker = listing[-1].get('version_id')
             listing = get_container(url, token, container, marker, limit,
-                                    prefix, delimiter, end_marker, path,
-                                    http_conn, service_token=service_token,
+                                    prefix, delimiter, end_marker,
+                                    version_marker, path, http_conn,
+                                    service_token=service_token,
                                     headers=headers)[1]
             if listing:
                 rv[1].extend(listing)
@@ -979,6 +983,8 @@ def get_container(url, token, container, marker=None, limit=None,
         qs += '&delimiter=%s' % quote(delimiter)
     if end_marker:
         qs += '&end_marker=%s' % quote(end_marker)
+    if version_marker:
+        qs += '&version_marker=%s' % quote(version_marker)
     if path:
         qs += '&path=%s' % quote(path)
     if query_string:
@@ -1816,15 +1822,17 @@ class Connection(object):
         return self._retry(None, head_container, container, headers=headers)
 
     def get_container(self, container, marker=None, limit=None, prefix=None,
-                      delimiter=None, end_marker=None, path=None,
-                      full_listing=False, headers=None, query_string=None):
+                      delimiter=None, end_marker=None, version_marker=None,
+                      path=None, full_listing=False, headers=None,
+                      query_string=None):
         """Wrapper for :func:`get_container`"""
         # TODO(unknown): With full_listing=True this will restart the entire
         # listing with each retry. Need to make a better version that just
         # retries where it left off.
         return self._retry(None, get_container, container, marker=marker,
                            limit=limit, prefix=prefix, delimiter=delimiter,
-                           end_marker=end_marker, path=path,
+                           end_marker=end_marker,
+                           version_marker=version_marker, path=path,
                            full_listing=full_listing, headers=headers,
                            query_string=query_string)
 
diff --git a/swiftclient/command_helpers.py b/swiftclient/command_helpers.py
index 49ccad1..f37040f 100644
--- a/swiftclient/command_helpers.py
+++ b/swiftclient/command_helpers.py
@@ -143,7 +143,11 @@ def print_container_stats(items, headers, output_manager):
 def stat_object(conn, options, container, obj):
     req_headers = split_request_headers(options.get('header', []))
 
-    headers = conn.head_object(container, obj, headers=req_headers)
+    query_string = None
+    if options.get('version_id') is not None:
+        query_string = 'version-id=%s' % options['version_id']
+    headers = conn.head_object(container, obj, headers=req_headers,
+                               query_string=query_string)
     items = []
     if options['verbose'] > 1:
         path = '%s/%s/%s' % (conn.url, container, obj)
diff --git a/swiftclient/service.py b/swiftclient/service.py
index 5292dc5..fb334fd 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -86,6 +86,9 @@ class SwiftError(Exception):
             value += " segment:%s" % self.segment
         return value
 
+    def __repr__(self):
+        return str(self)
+
 
 def process_options(options):
     # tolerate sloppy auth_version
@@ -186,6 +189,7 @@ _default_local_options = {
     'leave_segments': False,
     'changed': None,
     'skip_identical': False,
+    'version_id': None,
     'yes_all': False,
     'read_acl': None,
     'write_acl': None,
@@ -200,6 +204,7 @@ _default_local_options = {
     'meta': [],
     'prefix': None,
     'delimiter': None,
+    'versions': False,
     'fail_fast': False,
     'human': False,
     'dir_marker': False,
@@ -336,6 +341,20 @@ class SwiftPostObject(object):
         self.options = options
 
 
+class SwiftDeleteObject(object):
+    """
+    Class for specifying an object delete, allowing the headers/metadata to be
+    specified separately for each individual object.
+    """
+    def __init__(self, object_name, options=None):
+        if not (isinstance(object_name, string_types) and object_name):
+            raise SwiftError(
+                "Object names must be specified as non-empty strings"
+            )
+        self.object_name = object_name
+        self.options = options
+
+
 class SwiftCopyObject(object):
     """
     Class for specifying an object copy,
@@ -489,6 +508,7 @@ class SwiftService(object):
 
                             {
                                 'human': False,
+                                'version_id': None,
                                 'header': []
                             }
 
@@ -871,6 +891,7 @@ class SwiftService(object):
                                 'long': False,
                                 'prefix': None,
                                 'delimiter': None,
+                                'versions': False,
                                 'header': []
                             }
 
@@ -967,13 +988,19 @@ class SwiftService(object):
     @staticmethod
     def _list_container_job(conn, container, options, result_queue):
         marker = options.get('marker', '')
+        version_marker = options.get('version_marker', '')
         error = None
         req_headers = split_headers(options.get('header', []))
+        if options.get('versions', False):
+            query_string = 'versions=true'
+        else:
+            query_string = None
         try:
             while True:
                 _, items = conn.get_container(
-                    container, marker=marker, prefix=options['prefix'],
-                    delimiter=options['delimiter'], headers=req_headers
+                    container, marker=marker, version_marker=version_marker,
+                    prefix=options['prefix'], delimiter=options['delimiter'],
+                    headers=req_headers, query_string=query_string
                 )
 
                 if not items:
@@ -991,6 +1018,7 @@ class SwiftService(object):
                 result_queue.put(res)
 
                 marker = items[-1].get('name', items[-1].get('subdir'))
+                version_marker = items[-1].get('version_id', '')
         except ClientException as err:
             traceback, err_time = report_traceback()
             logger.exception(err)
@@ -1016,6 +1044,7 @@ class SwiftService(object):
             'prefix': options['prefix'],
             'success': False,
             'marker': marker,
+            'version_marker': version_marker,
             'error': error[0],
             'traceback': error[1],
             'error_timestamp': error[2]
@@ -1042,6 +1071,7 @@ class SwiftService(object):
                                 'no_download': False,
                                 'header': [],
                                 'skip_identical': False,
+                                'version_id': None,
                                 'out_directory': None,
                                 'checksum': True,
                                 'out_file': None,
@@ -1151,6 +1181,9 @@ class SwiftService(object):
             get_args = {'resp_chunk_size': DISK_BUFFER,
                         'headers': req_headers,
                         'response_dict': results_dict}
+            if options.get('version_id') is not None:
+                get_args['query_string'] = (
+                    'version-id=%s' % options['version_id'])
             if options['skip_identical']:
                 # Assume the file is a large object; if we're wrong, the query
                 # string is ignored and the If-None-Match header will trigger
@@ -2337,14 +2370,28 @@ class SwiftService(object):
         of objects.
 
         :param container: The container to delete or delete from.
-        :param objects: The list of objects to delete.
+        :param objects: A list of object names (strings) or SwiftDeleteObject
+                        instances containing an object name, and an
+                        options dict (can be None) to override the options for
+                        that individual delete operation::
+
+                            [
+                                'object_name',
+                                SwiftDeleteObject('object_name',
+                                                  options={...}),
+                                ...
+                            ]
+
+                        The options dict is described below.
         :param options: A dictionary containing options to override the global
                         options specified during the service object creation::
 
                             {
                                 'yes_all': False,
                                 'leave_segments': False,
+                                'version_id': None,
                                 'prefix': None,
+                                'versions': False,
                                 'header': [],
                             }
 
@@ -2364,23 +2411,28 @@ class SwiftService(object):
 
         if container is not None:
             if objects is not None:
+                delete_objects = self._make_delete_objects(objects)
                 if options['prefix']:
-                    objects = [obj for obj in objects
-                               if obj.startswith(options['prefix'])]
+                    delete_objects = [
+                        obj for obj in delete_objects
+                        if obj.object_name.startswith(options['prefix'])]
                 rq = Queue()
                 obj_dels = {}
 
-                bulk_page_size = self._bulk_delete_page_size(objects)
+                bulk_page_size = self._bulk_delete_page_size(delete_objects)
                 if bulk_page_size > 1:
-                    page_at_a_time = n_at_a_time(objects, bulk_page_size)
+                    page_at_a_time = n_at_a_time(delete_objects,
+                                                 bulk_page_size)
                     for page_slice in page_at_a_time:
                         for obj_slice in n_groups(
                                 page_slice,
                                 self._options['object_dd_threads']):
-                            self._bulk_delete(container, obj_slice, options,
+                            object_names = [
+                                obj.object_name for obj in obj_slice]
+                            self._bulk_delete(container, object_names, options,
                                               obj_dels)
                 else:
-                    self._per_item_delete(container, objects, options,
+                    self._per_item_delete(container, delete_objects, options,
                                           obj_dels, rq)
 
                 # Start a thread to watch for delete results
@@ -2445,6 +2497,11 @@ class SwiftService(object):
             # Not many objects; may as well delete one-by-one
             return 1
 
+        if any(obj.options for obj in objects
+               if isinstance(obj, SwiftDeleteObject)):
+            # we can't do per option deletes for bulk
+            return 1
+
         try:
             cap_result = self.capabilities()
             if not cap_result['success']:
@@ -2463,9 +2520,11 @@ class SwiftService(object):
             return 1
 
     def _per_item_delete(self, container, objects, options, rdict, rq):
-        for obj in objects:
+        for delete_obj in objects:
+            obj = delete_obj.object_name
+            obj_options = dict(options, **delete_obj.options or {})
             obj_del = self.thread_manager.object_dd_pool.submit(
-                self._delete_object, container, obj, options,
+                self._delete_object, container, obj, obj_options,
                 results_queue=rq
             )
             obj_details = {'container': container, 'object': obj}
@@ -2500,6 +2559,24 @@ class SwiftService(object):
             results_queue.put(res)
         return res
 
+    @staticmethod
+    def _make_delete_objects(objects):
+        delete_objects = []
+
+        for o in objects:
+            if isinstance(o, string_types):
+                obj = SwiftDeleteObject(o)
+                delete_objects.append(obj)
+            elif isinstance(o, SwiftDeleteObject):
+                delete_objects.append(o)
+            else:
+                raise SwiftError(
+                    "The delete operation takes only strings or "
+                    "SwiftDeleteObjects as input",
+                    obj=o)
+
+        return delete_objects
+
     def _delete_object(self, conn, container, obj, options,
                        results_queue=None):
         _headers = {}
@@ -2511,7 +2588,7 @@ class SwiftService(object):
         }
         try:
             old_manifest = None
-            query_string = None
+            query_params = {}
 
             if not options['leave_segments']:
                 try:
@@ -2520,11 +2597,15 @@ class SwiftService(object):
                                                query_string='symlink=get')
                     old_manifest = headers.get('x-object-manifest')
                     if config_true_value(headers.get('x-static-large-object')):
-                        query_string = 'multipart-manifest=delete'
+                        query_params['multipart-manifest'] = 'delete'
                 except ClientException as err:
                     if err.http_status != 404:
                         raise
 
+            if options.get('version_id') is not None:
+                query_params['version-id'] = options['version_id']
+            query_string = '&'.join('%s=%s' % (k, v) for (k, v)
+                                    in sorted(query_params.items()))
             results_dict = {}
             conn.delete_object(container, obj,
                                headers=_headers,
@@ -2611,12 +2692,17 @@ class SwiftService(object):
         try:
             for part in self.list(container=container, options=options):
                 if not part["success"]:
-
                     raise part["error"]
-
+                delete_objects = []
+                for item in part['listing']:
+                    delete_opts = {}
+                    if options.get('versions', False) and 'version_id' in item:
+                        delete_opts['version_id'] = item['version_id']
+                    delete_obj = SwiftDeleteObject(item['name'], delete_opts)
+                    delete_objects.append(delete_obj)
                 for res in self.delete(
                         container=container,
-                        objects=[o['name'] for o in part['listing']],
+                        objects=delete_objects,
                         options=options):
                     yield res
             if options['prefix']:
@@ -2679,7 +2765,9 @@ class SwiftService(object):
                         'No content received on account POST. '
                         'Is the bulk operations middleware enabled?')})
         except Exception as e:
-            res.update({'success': False, 'error': e})
+            traceback, err_time = report_traceback()
+            logger.exception(e)
+            res.update({'success': False, 'error': e, 'traceback': traceback})
 
         res.update({
             'action': 'bulk_delete',
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index d18fc9e..03a8fa6 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -65,7 +65,8 @@ st_delete_options = '''[--all] [--leave-segments]
                     [--container-threads <threads>]
                     [--header <header:value>]
                     [--prefix <prefix>]
-                    [<container> [<object>] [...]]
+                    [--versions]
+                    [<container> [<object>] [--version-id <version_id>] [...]]
 '''
 
 st_delete_help = '''
@@ -78,6 +79,7 @@ Positional arguments:
 
 Optional arguments:
   -a, --all             Delete all containers and objects.
+  --versions            Delete all versions
   --leave-segments      Do not delete segments of manifest objects.
   -H, --header <header:value>
                         Adds a custom request header to use for deleting
@@ -89,6 +91,8 @@ Optional arguments:
                         Number of threads to use for deleting containers.
                         Default is 10.
   --prefix <prefix>     Only delete objects beginning with <prefix>.
+  --version-id <version-id>
+                        Delete specific version of a versioned object.
 '''.strip("\n")
 
 
@@ -96,10 +100,15 @@ def st_delete(parser, args, output_manager, return_parser=False):
     parser.add_argument(
         '-a', '--all', action='store_true', dest='yes_all',
         default=False, help='Delete all containers and objects.')
+    parser.add_argument('--versions', action='store_true',
+                        help='delete all versions')
     parser.add_argument(
         '-p', '--prefix', dest='prefix',
         help='Only delete items beginning with <prefix>.')
     parser.add_argument(
+        '--version-id', action='store', default=None,
+        help='Delete a specific version of a versioned object')
+    parser.add_argument(
         '-H', '--header', action='append', dest='header',
         default=[],
         help='Adds a custom request header to use for deleting objects '
@@ -128,6 +137,10 @@ def st_delete(parser, args, output_manager, return_parser=False):
                              BASENAME, st_delete_options,
                              st_delete_help)
         return
+    if options['versions'] and len(args) >= 2:
+        exit('--versions option not allowed for object deletes')
+    if options['version_id'] and len(args) < 2:
+        exit('--version-id option only allowed for object deletes')
 
     if options['object_threads'] <= 0:
         output_manager.error(
@@ -227,6 +240,7 @@ st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>]
                       [--object-threads <threads>] [--ignore-checksum]
                       [--container-threads <threads>] [--no-download]
                       [--skip-identical] [--remove-prefix]
+                      [--version-id <version_id>]
                       [--header <header:value>] [--no-shuffle]
                       [<container> [<object>] [...]]
 '''
@@ -271,6 +285,8 @@ Optional arguments:
                         Example: --header "content-type:text/plain"
   --skip-identical      Skip downloading files that are identical on both
                         sides.
+  --version-id <version-id>
+                        Download specific version of a versioned object.
   --ignore-checksum     Turn off checksum validation for downloads.
   --no-shuffle          By default, when downloading a complete account or
                         container, download order is randomised in order to
@@ -333,6 +349,9 @@ def st_download(parser, args, output_manager, return_parser=False):
         default=False, help='Skip downloading files that are identical on '
         'both sides.')
     parser.add_argument(
+        '--version-id', action='store', default=None,
+        help='Download a specific version of a versioned object')
+    parser.add_argument(
         '--ignore-checksum', action='store_false', dest='checksum',
         default=True, help='Turn off checksum validation for downloads.')
     parser.add_argument(
@@ -372,6 +391,8 @@ def st_download(parser, args, output_manager, return_parser=False):
         output_manager.error('Usage: %s download %s\n%s', BASENAME,
                              st_download_options, st_download_help)
         return
+    if options['version_id'] and len(args) < 2:
+        exit('--version-id option only allowed for object downloads')
 
     if options['object_threads'] <= 0:
         output_manager.error(
@@ -479,7 +500,7 @@ def st_download(parser, args, output_manager, return_parser=False):
 
 st_list_options = '''[--long] [--lh] [--totals] [--prefix <prefix>]
                   [--delimiter <delimiter>] [--header <header:value>]
-                  [<container>]
+                  [--versions] [<container>]
 '''
 
 st_list_help = '''
@@ -499,6 +520,8 @@ Optional arguments:
                         Roll up items with the given delimiter. For containers
                         only. See OpenStack Swift API documentation for what
                         this means.
+  -j, --json            Display listing information in json
+  --versions            Display listing information for all versions
   -H, --header <header:value>
                         Adds a custom request header to use for listing.
 '''.strip('\n')
@@ -579,6 +602,8 @@ def st_list(parser, args, output_manager, return_parser=False):
              'what this means.')
     parser.add_argument('-j', '--json', action='store_true',
                         help='print listing information in json')
+    parser.add_argument('--versions', action='store_true',
+                        help='display all versions')
     parser.add_argument(
         '-H', '--header', action='append', dest='header',
         default=[],
@@ -592,6 +617,8 @@ def st_list(parser, args, output_manager, return_parser=False):
     args = args[1:]
     if options['delimiter'] and not args:
         exit('-d option only allowed for container listings')
+    if options['versions'] and not args:
+        exit('--versions option only allowed for container listings')
 
     human = options.pop('human')
     if human:
@@ -642,6 +669,7 @@ def st_list(parser, args, output_manager, return_parser=False):
 
 
 st_stat_options = '''[--lh] [--header <header:value>]
+                  [--version-id <version_id>]
                   [<container> [<object>]]
 '''
 
@@ -655,6 +683,8 @@ Positional arguments:
 Optional arguments:
   --lh                  Report sizes in human readable format similar to
                         ls -lh.
+  --version-id <version-id>
+                        Report stat of specific version of a versioned object.
   -H, --header <header:value>
                         Adds a custom request header to use for stat.
 '''.strip('\n')
@@ -665,6 +695,9 @@ def st_stat(parser, args, output_manager, return_parser=False):
         '--lh', dest='human', action='store_true', default=False,
         help='Report sizes in human readable format similar to ls -lh.')
     parser.add_argument(
+        '--version-id', action='store', default=None,
+        help='Report stat of a specific version of a versioned object')
+    parser.add_argument(
         '-H', '--header', action='append', dest='header',
         default=[],
         help='Adds a custom request header to use for stat.')
@@ -675,6 +708,8 @@ def st_stat(parser, args, output_manager, return_parser=False):
 
     options, args = parse_args(parser, args)
     args = args[1:]
+    if options['version_id'] and len(args) < 2:
+        exit('--version-id option only allowed for object stats')
 
     with SwiftService(options=options) as swift:
         try: