From 69bf4634b972ef2ee0ec1f015d71223203f0bb1a Mon Sep 17 00:00:00 2001 From: Cheng Li Date: Tue, 31 May 2016 14:32:58 +0800 Subject: Add an option: disable etag check on downloads This patch is to add an option of disable etag check on downloads. Change-Id: I9ad389dd691942dea6db470ca3f0543eb6e9703e Closes-bug: #1581147 --- doc/manpages/swift.1 | 1 + doc/source/cli.rst | 3 ++- doc/source/service-api.rst | 3 +++ swiftclient/service.py | 8 +++++--- swiftclient/shell.py | 6 +++++- tests/unit/test_service.py | 9 +++++++++ tests/unit/test_shell.py | 18 ++++++++++++++++++ 7 files changed, 43 insertions(+), 5 deletions(-) diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index b9f99c4..575adbe 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -86,6 +86,7 @@ container, or a list of objects depending on the args given. For a single object download, you may use the \-o [\-\-output] option to redirect the output to a specific file or if "-" then just redirect to stdout or with \-\-no-download actually not to write anything to disk. +The \-\-ignore-checksum is an option that turn off checksum validation. You can specify optional headers with the repeatable cURL-like option \-H [\-\-header]. For more details and options see swift download \-\-help. .RE diff --git a/doc/source/cli.rst b/doc/source/cli.rst index 12de02f..bbbfa3d 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -186,7 +186,8 @@ Download container, or a list of objects depending on the arguments given. For a single object download, you may use the ``-o `` or ``--output `` option to redirect the output to a specific file or ``-`` to - redirect to stdout. You can specify optional headers with the repeatable + redirect to stdout. The ``--ignore-checksum`` is an option that turn off + checksum validation. You can specify optional headers with the repeatable cURL-like option ``-H [--header ]``. Delete diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst index 7d65fd1..52f5857 100644 --- a/doc/source/service-api.rst +++ b/doc/source/service-api.rst @@ -207,6 +207,9 @@ Options Affects uploads, and allows empty 'pseudofolder' objects to be created when the source of an upload is ``None``. + ``checksum``: ``True`` + Affects uploads and downloads, and means if check md5 for downloads/uploads. + ``shuffle``: ``False`` When downloading objects, the default behaviour of the CLI is to shuffle lists of objects in order to spread the load on storage drives when multiple diff --git a/swiftclient/service.py b/swiftclient/service.py index 12d3f21..6ccba55 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -336,7 +336,7 @@ class _SwiftReader(object): errors on failures caused by either invalid md5sum or size of the data read. """ - def __init__(self, path, body, headers): + def __init__(self, path, body, headers, checksum=True): self._path = path self._body = body self._actual_read = 0 @@ -345,7 +345,7 @@ class _SwiftReader(object): self._expected_etag = headers.get('etag') if ('x-object-manifest' not in headers - and 'x-static-large-object' not in headers): + and 'x-static-large-object' not in headers and checksum): self._actual_md5 = md5() if 'content-length' in headers: @@ -980,6 +980,7 @@ class SwiftService(object): 'header': [], 'skip_identical': False, 'out_directory': None, + 'checksum': True, 'out_file': None, 'remove_prefix': False, 'shuffle' : False @@ -1135,7 +1136,8 @@ class SwiftService(object): headers_receipt = time() - obj_body = _SwiftReader(path, body, headers) + obj_body = _SwiftReader(path, body, headers, + options.get('checksum', True)) no_file = options['no_download'] if out_file == "-" and not no_file: diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 5eafe0b..5ba11ff 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -204,7 +204,7 @@ def st_delete(parser, args, output_manager): st_download_options = '''[--all] [--marker ] [--prefix ] [--output ] [--output-dir ] - [--object-threads ] + [--object-threads ] [--ignore-checksum] [--container-threads ] [--no-download] [--skip-identical] [--remove-prefix] [--header ] [--no-shuffle] @@ -251,6 +251,7 @@ Optional arguments: Example --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. + --ignore-checksum Turn off checksum validation for downloads. --no-shuffle By default, when downloading a complete account or container, download order is randomised in order to reduce the load on individual drives when multiple @@ -308,6 +309,9 @@ def st_download(parser, args, output_manager): '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip downloading files that are identical on ' 'both sides.') + parser.add_argument( + '--ignore-checksum', action='store_false', dest='checksum', + default=True, help='Turn off checksum validation for downloads.') parser.add_argument( '--no-shuffle', action='store_false', dest='shuffle', default=True, help='By default, download order is randomised in order ' diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index cce8c7a..75f8961 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -103,6 +103,15 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._expected_etag, None) self.assertEqual(sr._actual_md5, None) + def test_create_with_ignore_checksum(self): + # md5 should not be initialized if checksum is False + sr = self.sr('path', 'body', {}, False) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertEqual(sr._content_length, None) + self.assertEqual(sr._expected_etag, None) + self.assertEqual(sr._actual_md5, None) + def test_create_with_content_length(self): sr = self.sr('path', 'body', {'content-length': 5}) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index d82def6..9001b45 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -367,6 +367,24 @@ class TestShell(unittest.TestCase): mock_open.assert_called_with('object', 'wb') self.assertEqual([], makedirs.mock_calls) + # Test downloading without md5 checks + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + with mock.patch(BUILTIN_OPEN) as mock_open, mock.patch( + 'swiftclient.service._SwiftReader') as sr: + argv = ["", "download", "container", "object", "--ignore-check"] + swiftclient.shell.main(argv) + connection.return_value.get_object.assert_called_with( + 'container', 'object', headers={}, resp_chunk_size=65536, + response_dict={}) + mock_open.assert_called_with('object', 'wb') + sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) + self.assertEqual([], makedirs.mock_calls) + # Test downloading single object to stdout objcontent = six.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ -- cgit v1.2.1