From 3a5a25fe981817ba0e550d39d6e9863fa1539588 Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Thu, 3 Mar 2016 17:22:33 +0000 Subject: Add new doc structure and contents for swiftclient As a result of the Hackathon we have produced a new documentation structure for the python-swiftclient. This patch introduces the new structure and adds the required content. The intention is to document the CLI, the SwiftService and Connection API. Importantly, we also provide guidance on important considerations when using a swift object store, such as which aspect of the python-swiftclient to use for various use cases, common authentication patterns and some useful examples. Co-Authored-By: Alexandra Settle Co-Authored-By: Mohit Motiani Co-Authored-By: Hisashi Osanai Change-Id: I9eb41f8e9137efa66cead67dc264a76a3c03fbda --- examples/capabilities.py | 20 ++++++++++++++ examples/delete.py | 34 +++++++++++++++++++++++ examples/download.py | 37 +++++++++++++++++++++++++ examples/list.py | 32 ++++++++++++++++++++++ examples/post.py | 31 +++++++++++++++++++++ examples/stat.py | 25 +++++++++++++++++ examples/upload.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 250 insertions(+) create mode 100644 examples/capabilities.py create mode 100644 examples/delete.py create mode 100644 examples/download.py create mode 100644 examples/list.py create mode 100644 examples/post.py create mode 100644 examples/stat.py create mode 100644 examples/upload.py (limited to 'examples') diff --git a/examples/capabilities.py b/examples/capabilities.py new file mode 100644 index 0000000..024ad6f --- /dev/null +++ b/examples/capabilities.py @@ -0,0 +1,20 @@ +import logging + +from swiftclient.exceptions import ClientException +from swiftclient.service import SwiftService + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +with SwiftService() as swift: + try: + capabilities_result = swift.capabilities() + capabilities = capabilities_result['capabilities'] + if 'slo' in capabilities: + print('SLO is supported') + else: + print('SLO is not supported') + except ClientException as e: + logger.error(e.value) diff --git a/examples/delete.py b/examples/delete.py new file mode 100644 index 0000000..6979d9e --- /dev/null +++ b/examples/delete.py @@ -0,0 +1,34 @@ +import logging + +from swiftclient.service import SwiftService +from sys import argv + + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +_opts = {'object_dd_threads': 20} +container = argv[1] +objects = argv[2:] +with SwiftService(options=_opts) as swift: + del_iter = swift.delete(container=container, objects=objects) + for del_res in del_iter: + c = del_res.get('container', '') + o = del_res.get('object', '') + a = del_res.get('attempts') + if del_res['success'] and not del_res['action'] == 'bulk_delete': + rd = del_res.get('response_dict') + if rd is not None: + t = dict(rd.get('headers', {})) + if t: + print( + 'Successfully deleted {0}/{1} in {2} attempts ' + '(transaction id: {3})'.format(c, o, a, t) + ) + else: + print( + 'Successfully deleted {0}/{1} in {2} ' + 'attempts'.format(c, o, a) + ) diff --git a/examples/download.py b/examples/download.py new file mode 100644 index 0000000..5e3ebd0 --- /dev/null +++ b/examples/download.py @@ -0,0 +1,37 @@ +import logging + +from swiftclient.service import SwiftService, SwiftError +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +def is_png(obj): + return ( + obj["name"].lower().endswith('.png') or + obj["content_type"] == 'image/png' + ) + +container = argv[1] +with SwiftService() as swift: + try: + list_options = {"prefix": "archive_2016-01-01/"} + list_parts_gen = swift.list(container=container) + for page in list_parts_gen: + if page["success"]: + objects = [ + obj["name"] for obj in page["listing"] if is_png(obj) + ] + for down_res in swift.download( + container=container, + objects=objects): + if down_res['success']: + print("'%s' downloaded" % down_res['object']) + else: + print("'%s' download failed" % down_res['object']) + else: + raise page["error"] + except SwiftError as e: + logger.error(e.value) diff --git a/examples/list.py b/examples/list.py new file mode 100644 index 0000000..4b909d5 --- /dev/null +++ b/examples/list.py @@ -0,0 +1,32 @@ +import logging + +from swiftclient.service import SwiftService, SwiftError +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +container = argv[1] +minimum_size = 10*1024**2 +with SwiftService() as swift: + try: + list_parts_gen = swift.list(container=container) + for page in list_parts_gen: + if page["success"]: + for item in page["listing"]: + + i_size = int(item["bytes"]) + if i_size > minimum_size: + i_name = item["name"] + i_etag = item["hash"] + print( + "%s [size: %s] [etag: %s]" % + (i_name, i_size, i_etag) + ) + else: + raise page["error"] + + except SwiftError as e: + logger.error(e.value) diff --git a/examples/post.py b/examples/post.py new file mode 100644 index 0000000..c734543 --- /dev/null +++ b/examples/post.py @@ -0,0 +1,31 @@ +import logging + +from swiftclient.service import SwiftService, SwiftError +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +container = argv[1] +with SwiftService() as swift: + try: + list_options = {"prefix": "archive_2016-01-01/"} + list_parts_gen = swift.list(container=container) + for page in list_parts_gen: + if page["success"]: + objects = [obj["name"] for obj in page["listing"]] + post_options = {"header": "X-Delete-After:86400"} + for post_res in swift.post( + container=container, + objects=objects, + options=post_options): + if post_res['success']: + print("Object '%s' POST success" % post_res['object']) + else: + print("Object '%s' POST failed" % post_res['object']) + else: + raise page["error"] + except SwiftError as e: + logger.error(e.value) diff --git a/examples/stat.py b/examples/stat.py new file mode 100644 index 0000000..0905d1b --- /dev/null +++ b/examples/stat.py @@ -0,0 +1,25 @@ +import logging +import pprint + +from swiftclient.service import SwiftService +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +_opts = {'object_dd_threads': 20} +with SwiftService(options=_opts) as swift: + container = argv[1] + objects = argv[2:] + header_data = {} + stats_it = swift.stat(container=container, objects=objects) + for stat_res in stats_it: + if stat_res['success']: + header_data[stat_res['object']] = stat_res['headers'] + else: + logger.error( + 'Failed to retrieve stats for %s' % stat_res['object'] + ) + pprint.pprint(header_data) diff --git a/examples/upload.py b/examples/upload.py new file mode 100644 index 0000000..1b1e349 --- /dev/null +++ b/examples/upload.py @@ -0,0 +1,71 @@ +import logging + +from os.path import join, walk +from swiftclient.multithreading import OutputManager +from swiftclient.service import SwiftError, SwiftService, SwiftUploadObject +from sys import argv + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +_opts = {'object_uu_threads': 20} +dir = argv[1] +container = argv[2] +with SwiftService(options=_opts) as swift, OutputManager() as out_manager: + try: + # Collect all the files and folders in the given directory + objs = [] + dir_markers = [] + for (_dir, _ds, _fs) in walk(dir): + if not (_ds + _fs): + dir_markers.append(_dir) + else: + objs.extend([join(_dir, _f) for _f in _fs]) + + # Now that we've collected all the required files and dir markers + # build the ``SwiftUploadObject``s for the call to upload + objs = [ + SwiftUploadObject( + o, object_name=o.replace( + dir, 'my-%s-objects' % dir, 1 + ) + ) for o in objs + ] + dir_markers = [ + SwiftUploadObject( + None, object_name=d.replace( + dir, 'my-%s-objects' % dir, 1 + ), options={'dir_marker': True} + ) for d in dir_markers + ] + + # Schedule uploads on the SwiftService thread pool and iterate + # over the results + for r in swift.upload(container, objs + dir_markers): + if r['success']: + if 'object' in r: + print(r['object']) + elif 'for_object' in r: + print( + '%s segment %s' % (r['for_object'], + r['segment_index']) + ) + else: + error = r['error'] + if r['action'] == "create_container": + logger.warning( + 'Warning: failed to create container ' + "'%s'%s", container, error + ) + elif r['action'] == "upload_object": + logger.error( + "Failed to upload object %s to container %s: %s" % + (container, r['object'], error) + ) + else: + logger.error("%s" % error) + + except SwiftError as e: + logger.error(e.value) -- cgit v1.2.1