diff options
Diffstat (limited to 'tests/unit/test_service.py')
| -rw-r--r-- | tests/unit/test_service.py | 349 |
1 files changed, 305 insertions, 44 deletions
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 260f1cb..12fbaa0 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -36,6 +36,8 @@ from swiftclient.service import ( SwiftService, SwiftError, SwiftUploadObject ) +from tests.unit import utils as test_utils + clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -119,25 +121,36 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) - self.assertIsNotNone(sr._actual_md5) - self.assertIs(type(sr._actual_md5), self.md5_type) + self.assertIsNone(sr._actual_md5) def test_create_with_large_object_headers(self): # md5 should not be initialized if large object headers are present - sr = self.sr('path', 'body', {'x-object-manifest': 'test'}) + sr = self.sr('path', 'body', {'x-object-manifest': 'test', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + sr = self.sr('path', 'body', {'x-static-large-object': 'test', + 'etag': '"%s"' % ('0' * 32)}) self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) - sr = self.sr('path', 'body', {'x-static-large-object': 'test'}) + def test_create_with_content_range_header(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'content-range': 'bytes 0-3/10', + 'etag': '"%s"' % ('0' * 32)}) self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) def test_create_with_ignore_checksum(self): @@ -146,7 +159,7 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) def test_create_with_content_length(self): @@ -155,10 +168,9 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertEqual(sr._content_length, 5) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) - self.assertIsNotNone(sr._actual_md5) - self.assertIs(type(sr._actual_md5), self.md5_type) + self.assertIsNone(sr._actual_md5) # Check Contentlength raises error if it isn't an integer self.assertRaises(SwiftError, self.sr, 'path', 'body', @@ -175,11 +187,17 @@ class TestSwiftReader(unittest.TestCase): # Check error is raised if expected etag doesn't match calculated md5. # md5 for a SwiftReader that has done nothing is # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing - sr = self.sr('path', BytesIO(b'body'), {'etag': 'doesntmatch'}) + sr = self.sr('path', BytesIO(b'body'), + {'etag': md5(b'doesntmatch').hexdigest()}) self.assertRaises(SwiftError, _consume, sr) sr = self.sr('path', BytesIO(b'body'), - {'etag': '841a2d689ad86bd1611447453c22c6fc'}) + {'etag': md5(b'body').hexdigest()}) + _consume(sr) + + # Should still work if etag was quoted + sr = self.sr('path', BytesIO(b'body'), + {'etag': '"%s"' % md5(b'body').hexdigest()}) _consume(sr) # Check error is raised if SwiftReader doesn't read the same length @@ -191,11 +209,13 @@ class TestSwiftReader(unittest.TestCase): _consume(sr) # Check that the iterator generates expected length and etag values - sr = self.sr('path', ['abc'.encode()] * 3, {}) + sr = self.sr('path', ['abc'.encode()] * 3, + {'content-length': 9, + 'etag': md5('abc'.encode() * 3).hexdigest()}) _consume(sr) self.assertEqual(sr._actual_read, 9) self.assertEqual(sr._actual_md5.hexdigest(), - '97ac82a5b825239e782d0339e2d7b910') + md5('abc'.encode() * 3).hexdigest()) class _TestServiceBase(unittest.TestCase): @@ -1070,6 +1090,83 @@ class TestService(unittest.TestCase): self.assertEqual(upload_obj_resp['path'], obj['path']) self.assertTrue(mock_open.return_value.closed) + @mock.patch('swiftclient.service.Connection') + def test_upload_stream(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + segment_etag = md5(b'A' * 1024).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + segment_etag + options = {'use_slo': True, 'segment_size': 1024} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertFalse('error' in resp) + self.assertTrue(resp['success']) + self.assertEqual(5, len(responses)) + container_resp, segment_container_resp = responses[0:2] + segment_response = responses[2:4] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertTrue(upload_obj_resp['large_object']) + self.assertIn('manifest_response_dict', upload_obj_resp) + self.assertEqual(upload_obj_resp['manifest_response_dict'], {}) + for i, resp in enumerate(segment_response): + self.assertEqual(i, resp['segment_index']) + self.assertEqual(1024, resp['segment_size']) + self.assertEqual('d47b127bc2de2d687ddc82dac354c415', + resp['segment_etag']) + self.assertTrue(resp['segment_location'].endswith( + '/0000000%d' % i)) + self.assertTrue(resp['segment_location'].startswith( + '/container_segments/streamed')) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream_fits_in_one_segment(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + whole_etag = md5(b'A' * 2048).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + whole_etag + options = {'use_slo': True, 'segment_size': 10240} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertNotIn('error', resp) + self.assertTrue(resp['success']) + self.assertEqual(3, len(responses)) + container_resp, segment_container_resp = responses[0:2] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertFalse(upload_obj_resp['large_object']) + self.assertNotIn('manifest_response_dict', upload_obj_resp) + class TestServiceUpload(_TestServiceBase): @@ -1128,14 +1225,9 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='ใในใ/dummy.dat', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 10, - 'segment_container': None, - 'use_slo': False, - 'checksum': True}) + options=dict(s._options, + segment_size=10, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1213,6 +1305,141 @@ class TestServiceUpload(_TestServiceBase): self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) + def test_upload_stream_segment(self): + common_params = { + 'segment_container': 'segments', + 'segment_name': 'test_stream_2', + 'container': 'test_stream', + 'object': 'stream_object', + } + tests = [ + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1024}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 512}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 512).hexdigest()}}, + # 0-sized segment should not be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 0}, + 'put_object_args': {}, + 'expected': { + 'complete': True}}, + # 0-sized objects should be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 0, + 'content_size': 0}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'').hexdigest()}}, + # Test boundary conditions + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 1023}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 1023).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 2047}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 2047).hexdigest()}}, + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1025}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + ] + + for test_args in tests: + params = test_args['test_params'] + stream = test_utils.FakeStream(params['content_size']) + segment_size = params['segment_size'] + segment_index = params['segment_index'] + + def _fake_put_object(*args, **kwargs): + contents = args[2] + # Consume and compute md5 + return md5(contents).hexdigest() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _fake_put_object + + s = SwiftService() + resp = s._upload_stream_segment( + conn=mock_conn, + container=common_params['container'], + object_name=common_params['object'], + segment_container=common_params['segment_container'], + segment_name=common_params['segment_name'], + segment_size=segment_size, + segment_index=segment_index, + headers={}, + fd=stream) + expected_args = test_args['expected'] + put_args = test_args['put_object_args'] + expected_response = { + 'segment_size': min(len(stream), segment_size), + 'complete': expected_args['complete'], + 'success': True, + } + if len(stream) or segment_index == 0: + segment_location = '/%s/%s' % (put_args['container'], + put_args['object']) + expected_response.update( + {'segment_index': segment_index, + 'segment_location': segment_location, + 'segment_etag': expected_args['segment_etag'], + 'for_object': common_params['object']}) + mock_conn.put_object.assert_called_once_with( + put_args['container'], + put_args['object'], + mock.ANY, + content_length=min(len(stream), segment_size), + headers={'etag': expected_args['segment_etag']}, + response_dict=mock.ANY) + else: + self.assertEqual([], mock_conn.put_object.mock_calls) + expected_response.update( + {'segment_index': None, + 'segment_location': None, + 'segment_etag': None}) + self.assertEqual(expected_response, resp) + def test_etag_mismatch_with_ignore_checksum(self): def _consuming_conn(*a, **kw): contents = a[2] @@ -1332,12 +1559,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1387,12 +1610,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = float(r['headers']['x-object-meta-mtime']) self.assertEqual(mtime, expected_mtime) @@ -1434,12 +1653,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) self.assertIs(r['success'], False) self.assertIn('md5 mismatch', str(r.get('error'))) @@ -1933,7 +2148,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) written_content.write.assert_called_once_with(b'objcontent') mock_conn.get_object.assert_called_once_with( @@ -1977,7 +2192,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) mock_utime.assert_called_once_with( 'test_o', (1454113727.682512, 1454113727.682512)) written_content.write.assert_called_once_with(b'objcontent') @@ -2023,7 +2238,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) self.assertEqual(0, len(mock_utime.mock_calls)) written_content.write.assert_called_once_with(b'objcontent') @@ -2033,6 +2248,52 @@ class TestServiceDownload(_TestServiceBase): ) self.assertEqual(expected_r, actual_r) + def test_download_object_job_ignore_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + _opts['ignore_mtime'] = True + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual([], mock_utime.mock_calls) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() mock_conn.get_object = Mock(side_effect=self.exc) |