diff options
author | Tim Burke <tim.burke@gmail.com> | 2017-03-15 18:05:38 +0000 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2017-07-11 17:04:49 -0700 |
commit | 638d7c789cf3ccab61bf6af6fcab6e6d79b9e0a4 (patch) | |
tree | 9574c7dbbbf4f7760e87a48970e5b1823a0ad0c7 | |
parent | cde73c196d4f161b227fb924cd59cf02eaa33c03 (diff) | |
download | python-swiftclient-638d7c789cf3ccab61bf6af6fcab6e6d79b9e0a4.tar.gz |
Buffer reads from disk
Otherwise, Python defaults to 8k reads which seems kinda terrible.
Change-Id: I3160626e947083af487fd1c3cb0aa6a62646527b
Closes-Bug: #1671621
-rw-r--r-- | swiftclient/service.py | 19 | ||||
-rw-r--r-- | tests/unit/test_service.py | 8 | ||||
-rw-r--r-- | tests/unit/test_shell.py | 12 |
3 files changed, 20 insertions, 19 deletions
diff --git a/swiftclient/service.py b/swiftclient/service.py index 5a43bf2..5f032be 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -50,6 +50,7 @@ from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager +DISK_BUFFER = 2 ** 16 logger = logging.getLogger("swiftclient.service") @@ -1126,14 +1127,14 @@ class SwiftService(object): if options['skip_identical']: filename = out_file if out_file else path try: - fp = open(filename, 'rb') + fp = open(filename, 'rb', DISK_BUFFER) except IOError: pass else: with fp: md5sum = md5() while True: - data = fp.read(65536) + data = fp.read(DISK_BUFFER) if not data: break md5sum.update(data) @@ -1141,7 +1142,7 @@ class SwiftService(object): try: start_time = time() - get_args = {'resp_chunk_size': 65536, + get_args = {'resp_chunk_size': DISK_BUFFER, 'headers': req_headers, 'response_dict': results_dict} if options['skip_identical']: @@ -1224,10 +1225,10 @@ class SwiftService(object): if not no_file: if out_file: - fp = open(out_file, 'wb') + fp = open(out_file, 'wb', DISK_BUFFER) else: if basename(path): - fp = open(path, 'wb') + fp = open(path, 'wb', DISK_BUFFER) else: pseudodir = True @@ -1733,7 +1734,7 @@ class SwiftService(object): } fp = None try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) fp.seek(segment_start) contents = LengthWrapper(fp, segment_size, md5=options['checksum']) @@ -1811,7 +1812,7 @@ class SwiftService(object): def _is_identical(self, chunk_data, path): try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) except IOError: return False @@ -1820,7 +1821,7 @@ class SwiftService(object): to_read = chunk['bytes'] md5sum = md5() while to_read: - data = fp.read(min(65536, to_read)) + data = fp.read(min(DISK_BUFFER, to_read)) if not data: return False md5sum.update(data) @@ -2032,7 +2033,7 @@ class SwiftService(object): try: if path is not None: content_length = getsize(path) - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) contents = LengthWrapper(fp, content_length, md5=options['checksum']) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 6490b3f..54f0a11 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1951,7 +1951,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) written_content.write.assert_called_once_with(b'objcontent') mock_conn.get_object.assert_called_once_with( @@ -1995,7 +1995,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) mock_utime.assert_called_once_with( 'test_o', (1454113727.682512, 1454113727.682512)) written_content.write.assert_called_once_with(b'objcontent') @@ -2041,7 +2041,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) self.assertEqual(0, len(mock_utime.mock_calls)) written_content.write.assert_called_once_with(b'objcontent') @@ -2087,7 +2087,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) self.assertEqual([], mock_utime.mock_calls) written_content.write.assert_called_once_with(b'objcontent') diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index db96df7..4c47db1 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -473,7 +473,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) makedirs.reset_mock() @@ -490,7 +490,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) self.assertEqual([], makedirs.mock_calls) # Test downloading without md5 checks @@ -507,7 +507,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) self.assertEqual([], makedirs.mock_calls) @@ -553,7 +553,7 @@ class TestShell(unittest.TestCase): mock_shuffle.assert_any_call(['container']) mock_shuffle.assert_any_call(['object']) mock_shuffle.assert_any_call(['pseudo/']) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -577,7 +577,7 @@ class TestShell(unittest.TestCase): argv = ["", "download", "--all", "--no-shuffle"] swiftclient.shell.main(argv) self.assertEqual(0, mock_shuffle.call_count) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -610,7 +610,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([ mock.call('pseudo'), ], mock_mkdir.mock_calls) |