diff options
| -rwxr-xr-x | bin/swift | 126 | ||||
| -rw-r--r-- | swiftclient/command_helpers.py | 91 | ||||
| -rw-r--r-- | swiftclient/multithreading.py | 24 | ||||
| -rw-r--r-- | swiftclient/utils.py | 30 | ||||
| -rw-r--r-- | tests/test_command_helpers.py | 193 | ||||
| -rw-r--r-- | tests/test_swiftclient.py | 20 | ||||
| -rw-r--r-- | tests/test_utils.py | 119 |
7 files changed, 463 insertions, 140 deletions
@@ -33,7 +33,8 @@ except ImportError: import json from swiftclient import Connection, HTTPException -from swiftclient.utils import config_true_value +from swiftclient import command_helpers +from swiftclient.utils import config_true_value, prt_bytes from swiftclient.multithreading import MultiThreadingManager from swiftclient.exceptions import ClientException from swiftclient.version import version_info @@ -455,34 +456,6 @@ def st_download(parser, args, thread_manager): for obj in args[1:]: object_queue.put((args[0], obj)) - -def prt_bytes(bytes, human_flag): - """ - convert a number > 1024 to printable format, either in 4 char -h format as - with ls -lh or return as 12 char right justified string - """ - - if human_flag: - suffix = '' - mods = 'KMGTPEZY' - temp = float(bytes) - if temp > 0: - while (temp > 1023): - temp /= 1024.0 - suffix = mods[0] - mods = mods[1:] - if suffix != '': - if temp >= 10: - bytes = '%3d%s' % (temp, suffix) - else: - bytes = '%.1f%s' % (temp, suffix) - if suffix == '': # must be < 1024 - bytes = '%4s' % bytes - else: - bytes = '%12s' % bytes - - return(bytes) - st_list_options = '''[--long] [--lh] [--totals] [--container-threads <threads>] ''' @@ -628,34 +601,7 @@ def st_stat(parser, args, thread_manager): conn = get_conn(options) if not args: try: - headers = conn.head_account() - if options.verbose > 1: - thread_manager.print_msg(''' -StorageURL: %s -Auth Token: %s -'''.strip('\n'), conn.url, conn.token) - container_count = int(headers.get('x-account-container-count', 0)) - object_count = prt_bytes(headers.get('x-account-object-count', 0), - options.human).lstrip() - bytes_used = prt_bytes(headers.get('x-account-bytes-used', 0), - options.human).lstrip() - thread_manager.print_msg(''' - Account: %s -Containers: %d - Objects: %s - Bytes: %s'''.strip('\n'), conn.url.rsplit('/', 1)[-1], container_count, - object_count, bytes_used) - for key, value in headers.items(): - if key.startswith('x-account-meta-'): - thread_manager.print_msg( - '%10s: %s', - 'Meta %s' % key[len('x-account-meta-'):].title(), - value) - for key, value in headers.items(): - if not key.startswith('x-account-meta-') and key not in ( - 'content-length', 'date', 'x-account-container-count', - 'x-account-object-count', 'x-account-bytes-used'): - thread_manager.print_msg('%10s: %s', key.title(), value) + command_helpers.stat_account(conn, options, thread_manager) except ClientException as err: if err.http_status != 404: raise @@ -666,75 +612,15 @@ Containers: %d 'meant %r instead of %r.' % \ (args[0].replace('/', ' ', 1), args[0]) try: - headers = conn.head_container(args[0]) - object_count = prt_bytes( - headers.get('x-container-object-count', 0), - options.human).lstrip() - bytes_used = prt_bytes(headers.get('x-container-bytes-used', 0), - options.human).lstrip() - thread_manager.print_msg(''' - Account: %s -Container: %s - Objects: %s - Bytes: %s - Read ACL: %s -Write ACL: %s - Sync To: %s - Sync Key: %s'''.strip('\n'), conn.url.rsplit('/', 1)[-1], args[0], - object_count, bytes_used, - headers.get('x-container-read', ''), - headers.get('x-container-write', ''), - headers.get('x-container-sync-to', ''), - headers.get('x-container-sync-key', '')) - for key, value in headers.items(): - if key.startswith('x-container-meta-'): - thread_manager.print_msg( - '%9s: %s', - 'Meta %s' % key[len('x-container-meta-'):].title(), - value) - for key, value in headers.items(): - if not key.startswith('x-container-meta-') and key not in ( - 'content-length', 'date', 'x-container-object-count', - 'x-container-bytes-used', 'x-container-read', - 'x-container-write', 'x-container-sync-to', - 'x-container-sync-key'): - thread_manager.print_msg('%9s: %s', key.title(), value) + command_helpers.stat_container(conn, options, args, + thread_manager) except ClientException as err: if err.http_status != 404: raise thread_manager.error('Container %r not found', args[0]) elif len(args) == 2: try: - headers = conn.head_object(args[0], args[1]) - thread_manager.print_msg(''' - Account: %s - Container: %s - Object: %s - Content Type: %s'''.strip('\n'), conn.url.rsplit('/', 1)[-1], args[0], - args[1], headers.get('content-type')) - if 'content-length' in headers: - thread_manager.print_msg('Content Length: %s', - prt_bytes(headers['content-length'], - options.human).lstrip()) - if 'last-modified' in headers: - thread_manager.print_msg(' Last Modified: %s', - headers['last-modified']) - if 'etag' in headers: - thread_manager.print_msg(' ETag: %s', headers['etag']) - if 'x-object-manifest' in headers: - thread_manager.print_msg(' Manifest: %s', - headers['x-object-manifest']) - for key, value in headers.items(): - if key.startswith('x-object-meta-'): - thread_manager.print_msg( - '%14s: %s', - 'Meta %s' % key[len('x-object-meta-'):].title(), - value) - for key, value in headers.items(): - if not key.startswith('x-object-meta-') and key not in ( - 'content-type', 'content-length', 'last-modified', - 'etag', 'date', 'x-object-manifest'): - thread_manager.print_msg('%14s: %s', key.title(), value) + command_helpers.stat_object(conn, options, args, thread_manager) except ClientException as err: if err.http_status != 404: raise diff --git a/swiftclient/command_helpers.py b/swiftclient/command_helpers.py new file mode 100644 index 0000000..4e9c664 --- /dev/null +++ b/swiftclient/command_helpers.py @@ -0,0 +1,91 @@ +from swiftclient.utils import prt_bytes + + +def stat_account(conn, options, thread_manager): + headers = conn.head_account() + if options.verbose > 1: + thread_manager.print_items(( + ('StorageURL', conn.url), + ('Auth Token', conn.token), + )) + container_count = int(headers.get('x-account-container-count', 0)) + object_count = prt_bytes(headers.get('x-account-object-count', 0), + options.human).lstrip() + bytes_used = prt_bytes(headers.get('x-account-bytes-used', 0), + options.human).lstrip() + thread_manager.print_items(( + ('Account', conn.url.rsplit('/', 1)[-1]), + ('Containers', container_count), + ('Objects', object_count), + ('Bytes', bytes_used), + )) + thread_manager.print_headers(headers, + meta_prefix='x-account-meta-', + exclude_headers=( + 'content-length', 'date', + 'x-account-container-count', + 'x-account-object-count', + 'x-account-bytes-used')) + + +def stat_container(conn, options, args, thread_manager): + headers = conn.head_container(args[0]) + if options.verbose > 1: + path = '%s/%s' % (conn.url, args[0]) + thread_manager.print_items(( + ('URL', path), + ('Auth Token', conn.token), + )) + object_count = prt_bytes( + headers.get('x-container-object-count', 0), + options.human).lstrip() + bytes_used = prt_bytes(headers.get('x-container-bytes-used', 0), + options.human).lstrip() + thread_manager.print_items(( + ('Account', conn.url.rsplit('/', 1)[-1]), + ('Container', args[0]), + ('Objects', object_count), + ('Bytes', bytes_used), + ('Read ACL', headers.get('x-container-read', '')), + ('Write ACL', headers.get('x-container-write', '')), + ('Sync To', headers.get('x-container-sync-to', '')), + ('Sync Key', headers.get('x-container-sync-key', '')), + )) + thread_manager.print_headers(headers, + meta_prefix='x-container-meta-', + exclude_headers=( + 'content-length', 'date', + 'x-container-object-count', + 'x-container-bytes-used', + 'x-container-read', + 'x-container-write', + 'x-container-sync-to', + 'x-container-sync-key')) + + +def stat_object(conn, options, args, thread_manager): + headers = conn.head_object(args[0], args[1]) + if options.verbose > 1: + path = '%s/%s/%s' % (conn.url, args[0], args[1]) + thread_manager.print_items(( + ('URL', path), + ('Auth Token', conn.token), + )) + content_length = prt_bytes(headers.get('content-length', 0), + options.human).lstrip() + thread_manager.print_items(( + ('Account', conn.url.rsplit('/', 1)[-1]), + ('Container', args[0]), + ('Object', args[1]), + ('Content Type', headers.get('content-type')), + ('Content Length', content_length), + ('Last Modified', headers.get('last-modified')), + ('ETag', headers.get('etag')), + ('Manifest', headers.get('x-object-manifest')), + ), skip_missing=True) + thread_manager.print_headers(headers, + meta_prefix='x-object-meta-', + exclude_headers=( + 'content-type', 'content-length', + 'last-modified', 'etag', 'date', + 'x-object-manifest')) diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index 890a789..512c6e1 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -12,6 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +from itertools import chain import sys from time import sleep from Queue import Queue @@ -224,6 +225,29 @@ class MultiThreadingManager(object): msg = msg % fmt_args self.printer.queue.put(msg) + def print_items(self, items, offset=14, skip_missing=False): + lines = [] + template = '%%%ds: %%s' % offset + for k, v in items: + if skip_missing and not v: + continue + lines.append((template % (k, v)).rstrip()) + self.print_msg('\n'.join(lines)) + + def print_headers(self, headers, meta_prefix='', exclude_headers=None, + offset=14): + exclude_headers = exclude_headers or [] + meta_headers = [] + other_headers = [] + template = '%%%ds: %%s' % offset + for key, value in headers.items(): + if key.startswith(meta_prefix): + meta_key = 'Meta %s' % key[len(meta_prefix):].title() + meta_headers.append(template % (meta_key, value)) + elif key not in exclude_headers: + other_headers.append(template % (key.title(), value)) + self.print_msg('\n'.join(chain(meta_headers, other_headers))) + def error(self, msg, *fmt_args): if fmt_args: msg = msg % fmt_args diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 33d89a5..a038dcc 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -25,3 +25,33 @@ def config_true_value(value): """ return value is True or \ (isinstance(value, basestring) and value.lower() in TRUE_VALUES) + + +def prt_bytes(bytes, human_flag): + """ + convert a number > 1024 to printable format, either in 4 char -h format as + with ls -lh or return as 12 char right justified string + """ + + if human_flag: + suffix = '' + mods = list('KMGTPEZY') + temp = float(bytes) + if temp > 0: + while (temp > 1023): + try: + suffix = mods.pop(0) + except IndexError: + break + temp /= 1024.0 + if suffix != '': + if temp >= 10: + bytes = '%3d%s' % (temp, suffix) + else: + bytes = '%.1f%s' % (temp, suffix) + if suffix == '': # must be < 1024 + bytes = '%4s' % bytes + else: + bytes = '%12s' % bytes + + return(bytes) diff --git a/tests/test_command_helpers.py b/tests/test_command_helpers.py new file mode 100644 index 0000000..225805b --- /dev/null +++ b/tests/test_command_helpers.py @@ -0,0 +1,193 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from StringIO import StringIO +import mock +import testtools + +from swiftclient import command_helpers as h +from swiftclient.multithreading import MultiThreadingManager + + +class TestStatHelpers(testtools.TestCase): + + def setUp(self): + super(TestStatHelpers, self).setUp() + conn_attrs = { + 'url': 'http://storage/v1/a', + 'token': 'tk12345', + } + self.conn = mock.MagicMock(**conn_attrs) + self.options = mock.MagicMock(human=False, verbose=1) + self.stdout = StringIO() + self.stderr = StringIO() + self.thread_manager = MultiThreadingManager(self.stdout, self.stderr) + + def assertOut(self, expected): + real = self.stdout.getvalue() + # commonly if we strip of blank lines we have a match + try: + self.assertEqual(expected.strip('\n'), + real.strip('\n')) + except AssertionError: + # could be anything, try to find typos line by line + expected_lines = [line.lstrip() for line in + expected.splitlines() if line.strip()] + real_lines = [line.lstrip() for line in + real.splitlines() if line.strip()] + for expected, real in zip(expected_lines, real_lines): + self.assertEqual(expected, real) + # not a typo, might be an indent thing, hopefully you can spot it + raise + + def test_stat_account_human(self): + self.options.human = True + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.thread_manager as thread_manager: + h.stat_account(self.conn, self.options, thread_manager) + expected = """ + Account: a + Containers: 42 + Objects: 976K + Bytes: 1.0G +""" + self.assertOut(expected) + + def test_stat_account_verbose(self): + self.options.verbose += 1 + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.thread_manager as thread_manager: + h.stat_account(self.conn, self.options, thread_manager) + expected = """ + StorageURL: http://storage/v1/a + Auth Token: tk12345 + Account: a + Containers: 42 + Objects: 1000000 + Bytes: 1073741824 +""" + self.assertOut(expected) + + def test_stat_container_human(self): + self.options.human = True + # stub head container request + stub_headers = { + 'x-container-object-count': 10 ** 6, + 'x-container-bytes-used': 2 ** 30, + } + self.conn.head_container.return_value = stub_headers + args = ('c',) + with self.thread_manager as thread_manager: + h.stat_container(self.conn, self.options, args, thread_manager) + expected = """ + Account: a + Container: c + Objects: 976K + Bytes: 1.0G + Read ACL: + Write ACL: + Sync To: + Sync Key: +""" + self.assertOut(expected) + + def test_stat_container_verbose(self): + self.options.verbose += 1 + # stub head container request + stub_headers = { + 'x-container-object-count': 10 ** 6, + 'x-container-bytes-used': 2 ** 30, + } + self.conn.head_container.return_value = stub_headers + args = ('c',) + with self.thread_manager as thread_manager: + h.stat_container(self.conn, self.options, args, thread_manager) + expected = """ + URL: http://storage/v1/a/c + Auth Token: tk12345 + Account: a + Container: c + Objects: 1000000 + Bytes: 1073741824 + Read ACL: + Write ACL: + Sync To: + Sync Key: +""" + self.assertOut(expected) + + def test_stat_object_human(self): + self.options.human = True + # stub head object request + stub_headers = { + 'content-length': 2 ** 20, + 'x-object-meta-color': 'blue', + 'etag': '68b329da9893e34099c7d8ad5cb9c940', + 'content-encoding': 'gzip', + } + self.conn.head_object.return_value = stub_headers + args = ('c', 'o') + with self.thread_manager as thread_manager: + h.stat_object(self.conn, self.options, args, thread_manager) + expected = """ + Account: a + Container: c + Object: o +Content Length: 1.0M + ETag: 68b329da9893e34099c7d8ad5cb9c940 + Meta Color: blue +Content-Encoding: gzip +""" + self.assertOut(expected) + + def test_stat_object_verbose(self): + self.options.verbose += 1 + # stub head object request + stub_headers = { + 'content-length': 2 ** 20, + 'x-object-meta-color': 'blue', + 'etag': '68b329da9893e34099c7d8ad5cb9c940', + 'content-encoding': 'gzip', + } + self.conn.head_object.return_value = stub_headers + args = ('c', 'o') + with self.thread_manager as thread_manager: + h.stat_object(self.conn, self.options, args, thread_manager) + expected = """ + URL: http://storage/v1/a/c/o + Auth Token: tk12345 + Account: a + Container: c + Object: o +Content Length: 1048576 + ETag: 68b329da9893e34099c7d8ad5cb9c940 + Meta Color: blue +Content-Encoding: gzip +""" + self.assertOut(expected) diff --git a/tests/test_swiftclient.py b/tests/test_swiftclient.py index 6cf3c11..ada0b0c 100644 --- a/tests/test_swiftclient.py +++ b/tests/test_swiftclient.py @@ -26,7 +26,6 @@ from urlparse import urlparse from .utils import fake_http_connect, fake_get_keystoneclient_2_0 from swiftclient import client as c -from swiftclient import utils as u class TestClientException(testtools.TestCase): @@ -96,25 +95,6 @@ class TestJsonImport(testtools.TestCase): self.assertEqual(loads, c.json_loads) -class TestConfigTrueValue(testtools.TestCase): - - def test_TRUE_VALUES(self): - for v in u.TRUE_VALUES: - self.assertEqual(v, v.lower()) - - def test_config_true_value(self): - orig_trues = u.TRUE_VALUES - try: - u.TRUE_VALUES = 'hello world'.split() - for val in 'hello world HELLO WORLD'.split(): - self.assertTrue(u.config_true_value(val) is True) - self.assertTrue(u.config_true_value(True) is True) - self.assertTrue(u.config_true_value('foo') is False) - self.assertTrue(u.config_true_value(False) is False) - finally: - u.TRUE_VALUES = orig_trues - - class MockHttpTest(testtools.TestCase): def setUp(self): diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..b47d231 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,119 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import testtools + +from swiftclient import utils as u + + +class TestConfigTrueValue(testtools.TestCase): + + def test_TRUE_VALUES(self): + for v in u.TRUE_VALUES: + self.assertEqual(v, v.lower()) + + def test_config_true_value(self): + orig_trues = u.TRUE_VALUES + try: + u.TRUE_VALUES = 'hello world'.split() + for val in 'hello world HELLO WORLD'.split(): + self.assertTrue(u.config_true_value(val) is True) + self.assertTrue(u.config_true_value(True) is True) + self.assertTrue(u.config_true_value('foo') is False) + self.assertTrue(u.config_true_value(False) is False) + finally: + u.TRUE_VALUES = orig_trues + + +class TestPrtBytes(testtools.TestCase): + + def test_zero_bytes(self): + bytes_ = 0 + raw = '0' + human = '0' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_byte(self): + bytes_ = 1 + raw = '1' + human = '1' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_less_than_one_k(self): + bytes_ = (2 ** 10) - 1 + raw = '1023' + human = '1023' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_k(self): + bytes_ = 2 ** 10 + raw = '1024' + human = '1.0K' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_decimal_k(self): + bytes_ = (3 * 2 ** 10) + 512 + raw = '3584' + human = '3.5K' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_bit_less_than_one_meg(self): + bytes_ = (2 ** 20) - (2 ** 10) + raw = '1047552' + human = '1023K' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_just_a_hair_less_than_one_meg(self): + bytes_ = (2 ** 20) - (2 ** 10) + 1 + raw = '1047553' + human = '1.0M' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_meg(self): + bytes_ = 2 ** 20 + raw = '1048576' + human = '1.0M' + self.assertEquals(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_ten_meg(self): + bytes_ = 10 * 2 ** 20 + human = '10M' + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_bit_less_than_ten_meg(self): + bytes_ = (10 * 2 ** 20) - (100 * 2 ** 10) + human = '9.9M' + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_just_a_hair_less_than_ten_meg(self): + bytes_ = (10 * 2 ** 20) - 1 + human = '10.0M' + self.assertEquals(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_yotta(self): + bytes_ = 42 * 2 ** 80 + self.assertEquals('42Y', u.prt_bytes(bytes_, True).lstrip()) + + def test_overflow(self): + bytes_ = 2 ** 90 + self.assertEquals('1024Y', u.prt_bytes(bytes_, True).lstrip()) |