diff options
author | Tim Burke <tim.burke@gmail.com> | 2021-09-21 16:02:47 -0700 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2021-09-21 16:06:56 -0700 |
commit | 373fa26ce1620d096918bd34ea4983b8ca96898c (patch) | |
tree | 6a04dd1f0f0607c8897e517050c29d544cd2b5af | |
parent | 5129b33505691f5c4a3e33f2878b157cbd5b4a62 (diff) | |
download | python-swiftclient-373fa26ce1620d096918bd34ea4983b8ca96898c.tar.gz |
Correctly aggregate totals for >10k items
Previously, we would write out totals for every page of listings, like
$ swift list sync --prefix=09-21 --total -l
80000000000
80000000000
80000000000
58096000000
Now, roll those all into a single total:
$ swift list sync --prefix=09-21 --total -l
298096000000
Change-Id: Icc265636815220e33e8c9eec0a3ab80e9f899038
-rwxr-xr-x | swiftclient/shell.py | 32 | ||||
-rw-r--r-- | test/unit/test_shell.py | 13 |
2 files changed, 26 insertions, 19 deletions
diff --git a/swiftclient/shell.py b/swiftclient/shell.py index cf90ffc..fed0ef9 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -531,8 +531,7 @@ Optional arguments: def st_list(parser, args, output_manager, return_parser=False): - def _print_stats(options, stats, human): - total_count = total_bytes = 0 + def _print_stats(options, stats, human, totals): container = stats.get("container", None) for item in stats["listing"]: item_name = item.get('name') @@ -543,7 +542,7 @@ def st_list(parser, args, output_manager, return_parser=False): item_bytes = item.get('bytes') byte_str = prt_bytes(item_bytes, human) count = item.get('count') - total_count += count + totals['count'] += count try: meta = item.get('meta') utc = gmtime(float(meta.get('x-timestamp'))) @@ -578,17 +577,7 @@ def st_list(parser, args, output_manager, return_parser=False): output_manager.print_msg( "%s %10s %8s %24s %s", byte_str, date, xtime, content_type, item_name) - total_bytes += item_bytes - - # report totals - if options['long'] or human: - if not container: - output_manager.print_msg( - "%12s %s", prt_bytes(total_count, True), - prt_bytes(total_bytes, human)) - else: - output_manager.print_msg( - prt_bytes(total_bytes, human)) + totals['bytes'] += item_bytes parser.add_argument( '-l', '--long', dest='long', action='store_true', default=False, @@ -642,6 +631,7 @@ def st_list(parser, args, output_manager, return_parser=False): try: if not args: stats_parts_gen = swift.list() + container = None else: container = args[0] args = args[1:] @@ -667,12 +657,24 @@ def st_list(parser, args, output_manager, return_parser=False): sort_keys=True, indent=2) output_manager.print_msg('') return + + totals = {'count': 0, 'bytes': 0} for stats in stats_parts_gen: if stats["success"]: - _print_stats(options, stats, human) + _print_stats(options, stats, human, totals) else: raise stats["error"] + # report totals + if options['long'] or human: + if container is None: + output_manager.print_msg( + "%12s %s", prt_bytes(totals['count'], True), + prt_bytes(totals['bytes'], human)) + else: + output_manager.print_msg( + prt_bytes(totals['bytes'], human)) + except SwiftError as e: output_manager.error(e.value) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 84dd681..84793ae 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -547,9 +547,12 @@ class TestShell(unittest.TestCase): self.assertEqual(output.out, 'object_a\n') - # Test container listing with --long + # Test container listing with --long and multiple pages connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object_a', 'bytes': 0, + [None, [{'name': 'object_a', 'bytes': 3, + 'content_type': 'type/content', + 'last_modified': '123T456'}]], + [None, [{'name': 'object_b', 'bytes': 5, 'content_type': 'type/content', 'last_modified': '123T456'}]], [None, []], @@ -567,9 +570,11 @@ class TestShell(unittest.TestCase): connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, - ' 0 123 456' + ' 3 123 456' ' type/content object_a\n' - ' 0\n') + ' 5 123 456' + ' type/content object_b\n' + ' 8\n') @mock.patch('swiftclient.service.Connection') def test_list_container_with_headers(self, connection): |