diff options
author | Christopher Bartz <bartz@dkrz.de> | 2017-07-06 17:30:48 +0200 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2017-07-06 10:19:12 -0700 |
commit | cde73c196d4f161b227fb924cd59cf02eaa33c03 (patch) | |
tree | 6e4314a1d68974f4f00ab90b7162ab83e33f9e6b | |
parent | 1d57403668815ab8cef9d6598c06bf1c7e5355c0 (diff) | |
download | python-swiftclient-cde73c196d4f161b227fb924cd59cf02eaa33c03.tar.gz |
Option to ignore mtime metadata entry.
Currently, the swiftclient upload command passes a custom metadata
header for each object (called object-meta-mtime), whose value is
the current UNIX timestamp. When downloading such an object with the
swiftclient, the mtime header is parsed and passed as the atime and
mtime for the newly created file.
There are use-cases where this is not desired, for example when using
tmp or scratch directories in which files older than a specific date
are deleted. This commit provides a boolean option for ignoring the
mtime header.
Change-Id: If60b389aa910c6f1969b999b5d3b6d0940375686
-rw-r--r-- | doc/manpages/swift.1 | 3 | ||||
-rw-r--r-- | doc/source/cli.rst | 4 | ||||
-rw-r--r-- | swiftclient/service.py | 4 | ||||
-rwxr-xr-x | swiftclient/shell.py | 9 | ||||
-rw-r--r-- | tests/unit/test_service.py | 46 |
5 files changed, 64 insertions, 2 deletions
diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 1f288d6..d44a7d5 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -102,6 +102,9 @@ with \-\-no-download actually not to write anything to disk. The \-\-ignore-checksum is an option that turns off checksum validation. You can specify optional headers with the repeatable cURL-like option \-H [\-\-header]. For more details and options see swift download \-\-help. +The \-\-ignore\-mtime option ignores the x\-object\-meta\-mtime metadata entry +on the object (if present) and instead creates the downloaded files with +fresh atime and mtime values. .RE \fBdelete\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...] diff --git a/doc/source/cli.rst b/doc/source/cli.rst index 8d80d1b..8df95aa 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -188,7 +188,9 @@ Download option to redirect the output to a specific file or ``-`` to redirect to stdout. The ``--ignore-checksum`` is an option that turn off checksum validation. You can specify optional headers with the repeatable - cURL-like option ``-H [--header <name:value>]``. + cURL-like option ``-H [--header <name:value>]``. ``--ignore-mtime`` ignores the + ``x-object-meta-mtime`` metadata entry on the object (if present) and instead + creates the downloaded files with fresh atime and mtime values. Delete ------ diff --git a/swiftclient/service.py b/swiftclient/service.py index b9b843e..5a43bf2 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -203,6 +203,7 @@ _default_local_options = { 'shuffle': False, 'destination': None, 'fresh_metadata': False, + 'ignore_mtime': False, } POLICY = 'X-Storage-Policy' @@ -1240,7 +1241,8 @@ class SwiftService(object): bytes_read = obj_body.bytes_read() if fp is not None: fp.close() - if 'x-object-meta-mtime' in headers and not no_file: + if ('x-object-meta-mtime' in headers and not no_file + and not options['ignore_mtime']): try: mtime = float(headers['x-object-meta-mtime']) except ValueError: diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 841ed6e..19a224a 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -272,6 +272,9 @@ Optional arguments: script to multiple servers). Enable this option to submit download jobs to the thread pool in the order they are listed in the object store. + --ignore-mtime Ignore the 'X-Object-Meta-Mtime' header when + downloading an object. Instead, create atime and mtime + with fresh timestamps. '''.strip("\n") @@ -332,6 +335,12 @@ def st_download(parser, args, output_manager): 'nightly automated download script to multiple servers). Enable this ' 'option to submit download jobs to the thread pool in the order they ' 'are listed in the object store.') + parser.add_argument( + '--ignore-mtime', action='store_true', dest='ignore_mtime', + default=False, help='By default, the object-meta-mtime header is used ' + 'to store the access and modified timestamp for the downloaded file. ' + 'With this option, the header is ignored and the timestamps are ' + 'created freshly.') (options, args) = parse_args(parser, args) args = args[1:] if options['out_file'] == '-': diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index b759e6b..6490b3f 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -2051,6 +2051,52 @@ class TestServiceDownload(_TestServiceBase): ) self.assertEqual(expected_r, actual_r) + def test_download_object_job_ignore_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + _opts['ignore_mtime'] = True + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb') + self.assertEqual([], mock_utime.mock_calls) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() mock_conn.get_object = Mock(side_effect=self.exc) |