diff options
Diffstat (limited to 'test/unit')
| -rw-r--r-- | test/unit/__init__.py | 0 | ||||
| -rw-r--r-- | test/unit/test_authv1.py | 246 | ||||
| -rw-r--r-- | test/unit/test_command_helpers.py | 249 | ||||
| -rw-r--r-- | test/unit/test_multithreading.py | 240 | ||||
| -rw-r--r-- | test/unit/test_service.py | 2909 | ||||
| -rw-r--r-- | test/unit/test_shell.py | 3402 | ||||
| -rw-r--r-- | test/unit/test_swiftclient.py | 3328 | ||||
| -rw-r--r-- | test/unit/test_utils.py | 679 | ||||
| -rw-r--r-- | test/unit/utils.py | 582 |
9 files changed, 11635 insertions, 0 deletions
diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/unit/__init__.py diff --git a/test/unit/test_authv1.py b/test/unit/test_authv1.py new file mode 100644 index 0000000..2ddf24b --- /dev/null +++ b/test/unit/test_authv1.py @@ -0,0 +1,246 @@ +# Copyright 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +import datetime +import json +import mock +import unittest +from keystoneauth1 import plugin +from keystoneauth1 import loading +from keystoneauth1 import exceptions +from swiftclient import authv1 + + +class TestDataNoAccount(object): + options = dict( + auth_url='http://saio:8080/auth/v1.0', + username='test:tester', + password='testing') + storage_url = 'http://saio:8080/v1/AUTH_test' + expected_endpoint = storage_url + token = 'token' + + +class TestDataWithAccount(object): + options = dict( + auth_url='http://saio:8080/auth/v1.0', + username='test2:tester2', + project_name='SOME_other_account', + password='testing2') + storage_url = 'http://saio:8080/v1/AUTH_test2' + expected_endpoint = 'http://saio:8080/v1/SOME_other_account' + token = 'other_token' + + +class TestPluginLoading(TestDataNoAccount, unittest.TestCase): + def test_can_load(self): + loader = loading.get_plugin_loader('v1password') + self.assertIsInstance(loader, authv1.PasswordLoader) + + auth_plugin = loader.load_from_options(**self.options) + self.assertIsInstance(auth_plugin, authv1.PasswordPlugin) + + self.assertEqual(self.options['auth_url'], auth_plugin.auth_url) + self.assertEqual(self.options['username'], auth_plugin.user) + self.assertEqual(self.options.get('project_name'), auth_plugin.account) + self.assertEqual(self.options['password'], auth_plugin.key) + + def test_get_state(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.assertIsNone(auth_plugin.get_auth_state()) + + with mock.patch('swiftclient.authv1.time.time', return_value=1234.56): + auth_plugin.auth_ref = authv1.AccessInfoV1( + self.options['auth_url'], + self.storage_url, + self.options.get('project_name'), + self.options['username'], + self.token, + 60) + + expected = json.dumps({ + 'auth_url': self.options['auth_url'], + 'username': self.options['username'], + 'account': self.options.get('project_name'), + 'issued': 1234.56, + 'storage_url': self.storage_url, + 'auth_token': self.token, + 'expires': 1234.56 + 60, + }, sort_keys=True) + self.assertEqual(expected, auth_plugin.auth_ref.get_state()) + self.assertEqual(expected, auth_plugin.get_auth_state()) + + def test_set_state(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.assertIsNone(auth_plugin.auth_ref) + + auth_plugin.auth_ref = object() + auth_plugin.set_auth_state(None) + self.assertIsNone(auth_plugin.get_auth_state()) + + state = json.dumps({ + 'auth_url': self.options['auth_url'], + 'username': self.options['username'], + 'account': self.options.get('project_name'), + 'issued': 1234.56, + 'storage_url': self.storage_url, + 'auth_token': self.token, + 'expires': None, + }, sort_keys=True) + auth_plugin.set_auth_state(state) + self.assertIsInstance(auth_plugin.auth_ref, authv1.AccessInfoV1) + + self.assertEqual(self.options['username'], + auth_plugin.auth_ref.username) + self.assertEqual(self.options['auth_url'], + auth_plugin.auth_ref.auth_url) + self.assertEqual(self.storage_url, auth_plugin.auth_ref.storage_url) + self.assertEqual(self.options.get('project_name'), auth_plugin.account) + self.assertEqual(self.token, auth_plugin.auth_ref.auth_token) + self.assertEqual(1234.56, auth_plugin.auth_ref._issued) + self.assertIs(datetime.datetime, type(auth_plugin.auth_ref.issued)) + self.assertIsNone(auth_plugin.auth_ref._expires) + self.assertIsNone(auth_plugin.auth_ref.expires) + + +class TestPluginLoadingWithAccount(TestDataWithAccount, TestPluginLoading): + pass + + +class TestPlugin(TestDataNoAccount, unittest.TestCase): + def setUp(self): + self.mock_session = mock.MagicMock() + self.mock_response = self.mock_session.get.return_value + self.mock_response.status_code = 200 + self.mock_response.headers = { + 'X-Auth-Token': self.token, + 'X-Storage-Url': self.storage_url, + } + + def test_get_access(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + with mock.patch('swiftclient.authv1.time.time', return_value=1234.56): + access = auth_plugin.get_access(self.mock_session) + + self.assertEqual(self.mock_session.get.mock_calls, [mock.call( + self.options['auth_url'], authenticated=False, log=False, headers={ + 'X-Auth-User': self.options['username'], + 'X-Auth-Key': self.options['password'], + })]) + + self.assertEqual(self.options['username'], access.username) + # `openstack token issue` requires a user_id property + self.assertEqual(self.options['username'], access.user_id) + self.assertEqual(self.storage_url, access.storage_url) + self.assertEqual(self.token, access.auth_token) + self.assertEqual(1234.56, access._issued) + self.assertIs(datetime.datetime, type(auth_plugin.auth_ref.issued)) + self.assertIsNone(access.expires) + + # `openstack catalog list/show` require a catalog property + catalog = access.service_catalog.catalog + self.assertEqual('swift', catalog[0].get('name')) + self.assertEqual('object-store', catalog[0].get('type')) + self.assertIn('endpoints', catalog[0]) + self.assertIn(self.storage_url, [ + e.get('publicURL') for e in catalog[0]['endpoints']]) + + def test_get_access_with_expiry(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers['X-Auth-Token-Expires'] = '78.9' + with mock.patch('swiftclient.authv1.time.time', + return_value=1234.56) as mock_time: + access = auth_plugin.get_access(self.mock_session) + self.assertEqual(1234.56 + 78.9, access._expires) + self.assertIs(datetime.datetime, + type(auth_plugin.auth_ref.expires)) + + self.assertIs(True, access.will_expire_soon(90)) + self.assertIs(False, access.will_expire_soon(60)) + self.assertEqual(3, len(mock_time.mock_calls)) + + def test_get_access_bad_expiry(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers['X-Auth-Token-Expires'] = 'foo' + access = auth_plugin.get_access(self.mock_session) + self.assertIsNone(access.expires) + + self.assertIs(False, access.will_expire_soon(60)) + self.assertIs(False, access.will_expire_soon(1e20)) + + def test_get_access_bad_status(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.status_code = 401 + self.assertRaises(exceptions.InvalidResponse, + auth_plugin.get_access, self.mock_session) + + def test_get_access_missing_token(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers.pop('X-Auth-Token') + self.assertRaises(exceptions.InvalidResponse, + auth_plugin.get_access, self.mock_session) + + def test_get_access_accepts_storage_token(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers.pop('X-Auth-Token') + self.mock_response.headers['X-Storage-Token'] = 'yet another token' + access = auth_plugin.get_access(self.mock_session) + self.assertEqual('yet another token', access.auth_token) + + def test_get_access_missing_url(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers.pop('X-Storage-Url') + self.assertRaises(exceptions.InvalidResponse, + auth_plugin.get_access, self.mock_session) + + def test_get_endpoint(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + + object_store_endpoint = auth_plugin.get_endpoint( + self.mock_session, service_type='object-store') + self.assertEqual(object_store_endpoint, self.expected_endpoint) + + auth_endpoint = auth_plugin.get_endpoint( + self.mock_session, interface=plugin.AUTH_INTERFACE) + self.assertEqual(auth_endpoint, self.options['auth_url']) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint(self.mock_session) + self.assertEqual('public endpoint for None service not found', + str(exc_mgr.exception)) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint( + self.mock_session, service_type='identity', region_name='DFW') + self.assertEqual( + 'public endpoint for identity service in DFW region not found', + str(exc_mgr.exception)) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint( + self.mock_session, service_type='image', service_name='glance') + self.assertEqual( + 'public endpoint for image service named glance not found', + str(exc_mgr.exception)) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint( + self.mock_session, service_type='compute', service_name='nova', + region_name='IAD') + self.assertEqual('public endpoint for compute service named nova in ' + 'IAD region not found', str(exc_mgr.exception)) + + +class TestPluginWithAccount(TestDataWithAccount, TestPlugin): + pass diff --git a/test/unit/test_command_helpers.py b/test/unit/test_command_helpers.py new file mode 100644 index 0000000..24684ae --- /dev/null +++ b/test/unit/test_command_helpers.py @@ -0,0 +1,249 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from six import StringIO +import unittest + +from swiftclient import command_helpers as h +from swiftclient.multithreading import OutputManager + + +class TestStatHelpers(unittest.TestCase): + + def setUp(self): + super(TestStatHelpers, self).setUp() + conn_attrs = { + 'url': 'http://storage/v1/a', + 'token': 'tk12345', + } + self.conn = mock.MagicMock(**conn_attrs) + self.options = {'human': False, 'verbose': 1} + self.stdout = StringIO() + self.stderr = StringIO() + self.output_manager = OutputManager(self.stdout, self.stderr) + + def assertOut(self, expected): + real = self.stdout.getvalue() + # commonly if we strip of blank lines we have a match + try: + self.assertEqual(expected.strip('\n'), + real.strip('\n')) + except AssertionError: + # could be anything, try to find typos line by line + expected_lines = [line.lstrip() for line in + expected.splitlines() if line.strip()] + real_lines = [line.lstrip() for line in + real.splitlines() if line.strip()] + for expected, real in zip(expected_lines, real_lines): + self.assertEqual(expected, real) + # not a typo, might be an indent thing, hopefully you can spot it + raise + + def test_stat_account_human(self): + self.options['human'] = True + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ + Account: a +Containers: 42 + Objects: 976K + Bytes: 1.0G +""" + self.assertOut(expected) + + def test_stat_account_verbose(self): + self.options['verbose'] += 1 + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ +StorageURL: http://storage/v1/a +Auth Token: tk12345 + Account: a +Containers: 42 + Objects: 1000000 + Bytes: 1073741824 +""" + self.assertOut(expected) + + def test_stat_account_policy_stat(self): + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + 'x-account-storage-policy-nada-object-count': 1000000, + 'x-account-storage-policy-nada-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ + Account: a + Containers: 42 + Objects: 1000000 + Bytes: 1073741824 +Objects in policy "nada": 1000000 + Bytes in policy "nada": 1073741824 +""" + self.assertOut(expected) + + def test_stat_account_policy_stat_with_container_counts(self): + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + 'x-account-storage-policy-nada-container-count': 10, + 'x-account-storage-policy-nada-object-count': 1000000, + 'x-account-storage-policy-nada-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ + Account: a + Containers: 42 + Objects: 1000000 + Bytes: 1073741824 +Containers in policy "nada": 10 + Objects in policy "nada": 1000000 + Bytes in policy "nada": 1073741824 +""" + self.assertOut(expected) + + def test_stat_container_human(self): + self.options['human'] = True + # stub head container request + stub_headers = { + 'x-container-object-count': 10 ** 6, + 'x-container-bytes-used': 2 ** 30, + } + self.conn.head_container.return_value = stub_headers + args = ('c',) + with self.output_manager as output_manager: + items, headers = h.stat_container(self.conn, self.options, *args) + h.print_container_stats(items, headers, output_manager) + expected = """ + Account: a +Container: c + Objects: 976K + Bytes: 1.0G + Read ACL: +Write ACL: + Sync To: + Sync Key: +""" + self.assertOut(expected) + + def test_stat_container_verbose(self): + self.options['verbose'] += 1 + # stub head container request + stub_headers = { + 'x-container-object-count': 10 ** 6, + 'x-container-bytes-used': 2 ** 30, + } + self.conn.head_container.return_value = stub_headers + args = ('c',) + with self.output_manager as output_manager: + items, headers = h.stat_container(self.conn, self.options, *args) + h.print_container_stats(items, headers, output_manager) + expected = """ + URL: http://storage/v1/a/c +Auth Token: tk12345 + Account: a + Container: c + Objects: 1000000 + Bytes: 1073741824 + Read ACL: + Write ACL: + Sync To: + Sync Key: +""" + self.assertOut(expected) + + def test_stat_object_human(self): + self.options['human'] = True + # stub head object request + stub_headers = { + 'content-length': 2 ** 20, + 'x-object-meta-color': 'blue', + 'etag': '68b329da9893e34099c7d8ad5cb9c940', + 'content-encoding': 'gzip', + } + self.conn.head_object.return_value = stub_headers + args = ('c', 'o') + with self.output_manager as output_manager: + items, headers = h.stat_object(self.conn, self.options, *args) + h.print_object_stats(items, headers, output_manager) + expected = """ + Account: a + Container: c + Object: o + Content Length: 1.0M + ETag: 68b329da9893e34099c7d8ad5cb9c940 + Meta Color: blue +Content-Encoding: gzip +""" + self.assertOut(expected) + + def test_stat_object_verbose(self): + self.options['verbose'] += 1 + # stub head object request + stub_headers = { + 'content-length': 2 ** 20, + 'x-object-meta-color': 'blue', + 'etag': '68b329da9893e34099c7d8ad5cb9c940', + 'content-encoding': 'gzip', + } + self.conn.head_object.return_value = stub_headers + args = ('c', 'o') + with self.output_manager as output_manager: + items, headers = h.stat_object(self.conn, self.options, *args) + h.print_object_stats(items, headers, output_manager) + expected = """ + URL: http://storage/v1/a/c/o + Auth Token: tk12345 + Account: a + Container: c + Object: o + Content Length: 1048576 + ETag: 68b329da9893e34099c7d8ad5cb9c940 + Meta Color: blue +Content-Encoding: gzip +""" + self.assertOut(expected) diff --git a/test/unit/test_multithreading.py b/test/unit/test_multithreading.py new file mode 100644 index 0000000..8944d48 --- /dev/null +++ b/test/unit/test_multithreading.py @@ -0,0 +1,240 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import unittest +import threading +import six + +from concurrent.futures import as_completed +from six.moves.queue import Queue, Empty +from time import sleep + +from swiftclient import multithreading as mt +from .utils import CaptureStream + + +class ThreadTestCase(unittest.TestCase): + def setUp(self): + super(ThreadTestCase, self).setUp() + self.got_items = Queue() + self.got_args_kwargs = Queue() + self.starting_thread_count = threading.active_count() + + def _func(self, conn, item, *args, **kwargs): + self.got_items.put((conn, item)) + self.got_args_kwargs.put((args, kwargs)) + + if item == 'sleep': + sleep(1) + if item == 'go boom': + raise Exception('I went boom!') + + return 'success' + + def _create_conn(self): + return "This is a connection" + + def _create_conn_fail(self): + raise Exception("This is a failed connection") + + def assertQueueContains(self, queue, expected_contents): + got_contents = [] + try: + while True: + got_contents.append(queue.get(timeout=0.1)) + except Empty: + pass + if isinstance(expected_contents, set): + got_contents = set(got_contents) + self.assertEqual(expected_contents, got_contents) + + +class TestConnectionThreadPoolExecutor(ThreadTestCase): + def setUp(self): + super(TestConnectionThreadPoolExecutor, self).setUp() + self.input_queue = Queue() + self.stored_results = [] + + def tearDown(self): + super(TestConnectionThreadPoolExecutor, self).tearDown() + + def test_submit_good_connection(self): + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 1) + with ctpe as pool: + # Try submitting a job that should succeed + f = pool.submit(self._func, "succeed") + f.result() + self.assertQueueContains( + self.got_items, + [("This is a connection", "succeed")] + ) + + # Now a job that fails + try: + f = pool.submit(self._func, "go boom") + f.result() + except Exception as e: + self.assertEqual('I went boom!', str(e)) + else: + self.fail('I never went boom!') + + # Has the connection been returned to the pool? + f = pool.submit(self._func, "succeed") + f.result() + self.assertQueueContains( + self.got_items, + [ + ("This is a connection", "go boom"), + ("This is a connection", "succeed") + ] + ) + + def test_submit_bad_connection(self): + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn_fail, 1) + with ctpe as pool: + # Now a connection that fails + try: + f = pool.submit(self._func, "succeed") + f.result() + except Exception as e: + self.assertEqual('This is a failed connection', str(e)) + else: + self.fail('The connection did not fail') + + # Make sure we don't lock up on failed connections + try: + f = pool.submit(self._func, "go boom") + f.result() + except Exception as e: + self.assertEqual('This is a failed connection', str(e)) + else: + self.fail('The connection did not fail') + + def test_lazy_connections(self): + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) + with ctpe as pool: + # Submit multiple jobs sequentially - should only use 1 conn + f = pool.submit(self._func, "succeed") + f.result() + f = pool.submit(self._func, "succeed") + f.result() + f = pool.submit(self._func, "succeed") + f.result() + + expected_connections = [(0, "This is a connection")] + expected_connections.extend([(x, None) for x in range(1, 10)]) + + self.assertQueueContains( + pool._connections, expected_connections + ) + + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) + with ctpe as pool: + fs = [] + f1 = pool.submit(self._func, "sleep") + f2 = pool.submit(self._func, "sleep") + f3 = pool.submit(self._func, "sleep") + fs.extend([f1, f2, f3]) + + expected_connections = [ + (0, "This is a connection"), + (1, "This is a connection"), + (2, "This is a connection") + ] + expected_connections.extend([(x, None) for x in range(3, 10)]) + + for f in as_completed(fs): + f.result() + + self.assertQueueContains( + pool._connections, expected_connections + ) + + +class TestOutputManager(unittest.TestCase): + + def test_instantiation(self): + output_manager = mt.OutputManager() + + self.assertEqual(sys.stdout, output_manager.print_stream) + self.assertEqual(sys.stderr, output_manager.error_stream) + + def test_printers(self): + out_stream = CaptureStream(sys.stdout) + err_stream = CaptureStream(sys.stderr) + starting_thread_count = threading.active_count() + + with mt.OutputManager( + print_stream=out_stream, + error_stream=err_stream) as thread_manager: + + # Sanity-checking these gives power to the previous test which + # looked at the default values of thread_manager.print/error_stream + self.assertEqual(out_stream, thread_manager.print_stream) + self.assertEqual(err_stream, thread_manager.error_stream) + + # No printing has happened yet, so no new threads + self.assertEqual(starting_thread_count, + threading.active_count()) + + thread_manager.print_msg('one-argument') + thread_manager.print_msg('one %s, %d fish', 'fish', 88) + thread_manager.error('I have %d problems, but a %s is not one', + 99, u'\u062A\u062A') + thread_manager.print_msg('some\n%s\nover the %r', 'where', + u'\u062A\u062A') + thread_manager.error('one-error-argument') + thread_manager.error('Sometimes\n%.1f%% just\ndoes not\nwork!', + 3.14159) + thread_manager.print_raw( + u'some raw bytes: \u062A\u062A'.encode('utf-8')) + + thread_manager.print_items([ + ('key', 'value'), + ('object', u'O\u0308bject'), + ]) + + thread_manager.print_raw(b'\xffugly\xffraw') + + # Now we have a thread for error printing and a thread for + # normal print messages + self.assertEqual(starting_thread_count + 2, + threading.active_count()) + + # The threads should have been cleaned up + self.assertEqual(starting_thread_count, threading.active_count()) + + if six.PY3: + over_the = "over the '\u062a\u062a'\n" + else: + over_the = "over the u'\\u062a\\u062a'\n" + # We write to the CaptureStream so no decoding is performed + self.assertEqual(''.join([ + 'one-argument\n', + 'one fish, 88 fish\n', + 'some\n', 'where\n', + over_the, + u'some raw bytes: \u062a\u062a', + ' key: value\n', + u' object: O\u0308bject\n' + ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) + + self.assertEqual(''.join([ + u'I have 99 problems, but a \u062A\u062A is not one\n', + 'one-error-argument\n', + 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' + ]), err_stream.getvalue().decode('utf8')) + + self.assertEqual(3, thread_manager.error_count) diff --git a/test/unit/test_service.py b/test/unit/test_service.py new file mode 100644 index 0000000..ed3a2d6 --- /dev/null +++ b/test/unit/test_service.py @@ -0,0 +1,2909 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import unicode_literals +import contextlib +import mock +import os +import six +import tempfile +import unittest +import time + +from concurrent.futures import Future +from hashlib import md5 +from mock import Mock, PropertyMock +from six.moves.queue import Queue, Empty as QueueEmptyError +from six import BytesIO +from time import sleep + +import swiftclient +import swiftclient.utils as utils +from swiftclient.client import Connection, ClientException +from swiftclient.service import ( + SwiftService, SwiftError, SwiftUploadObject +) + +from test.unit import utils as test_utils + + +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' + + +if six.PY2: + import __builtin__ as builtins +else: + import builtins + + +class TestSwiftPostObject(unittest.TestCase): + + def setUp(self): + super(TestSwiftPostObject, self).setUp() + self.spo = swiftclient.service.SwiftPostObject + + def test_create(self): + spo = self.spo('obj_name') + + self.assertEqual(spo.object_name, 'obj_name') + self.assertIsNone(spo.options) + + def test_create_with_invalid_name(self): + # empty strings are not allowed as names + self.assertRaises(SwiftError, self.spo, '') + + # names cannot be anything but strings + self.assertRaises(SwiftError, self.spo, 1) + + +class TestSwiftCopyObject(unittest.TestCase): + + def setUp(self): + super(TestSwiftCopyObject, self).setUp() + self.sco = swiftclient.service.SwiftCopyObject + + def test_create(self): + sco = self.sco('obj_name') + + self.assertEqual(sco.object_name, 'obj_name') + self.assertIsNone(sco.destination) + self.assertFalse(sco.fresh_metadata) + + sco = self.sco('obj_name', + {'destination': '/dest', 'fresh_metadata': True}) + + self.assertEqual(sco.object_name, 'obj_name') + self.assertEqual(sco.destination, '/dest/obj_name') + self.assertTrue(sco.fresh_metadata) + + sco = self.sco('obj_name', + {'destination': '/dest/new_obj/a', + 'fresh_metadata': False}) + + self.assertEqual(sco.object_name, 'obj_name') + self.assertEqual(sco.destination, '/dest/new_obj/a') + self.assertFalse(sco.fresh_metadata) + + def test_create_with_invalid_name(self): + # empty strings are not allowed as names + self.assertRaises(SwiftError, self.sco, '') + + # names cannot be anything but strings + self.assertRaises(SwiftError, self.sco, 1) + + +class TestSwiftReader(unittest.TestCase): + + def setUp(self): + super(TestSwiftReader, self).setUp() + self.sr = swiftclient.service._SwiftReader + self.md5_type = type(md5()) + + def test_create(self): + sr = self.sr('path', 'body', {}) + + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + + self.assertIsNone(sr._actual_md5) + + def test_create_with_large_object_headers(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'x-object-manifest': 'test', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + sr = self.sr('path', 'body', {'x-static-large-object': 'test', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + def test_create_with_content_range_header(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'content-range': 'bytes 0-3/10', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + def test_create_with_ignore_checksum(self): + # md5 should not be initialized if checksum is False + sr = self.sr('path', 'body', {}, False) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + def test_create_with_content_length(self): + sr = self.sr('path', 'body', {'content-length': 5}) + + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertEqual(sr._content_length, 5) + self.assertFalse(sr._expected_md5) + + self.assertIsNone(sr._actual_md5) + + # Check Contentlength raises error if it isn't an integer + self.assertRaises(SwiftError, self.sr, 'path', 'body', + {'content-length': 'notanint'}) + + def test_iterator_usage(self): + def _consume(sr): + for _ in sr: + pass + + sr = self.sr('path', BytesIO(b'body'), {}) + _consume(sr) + + # Check error is raised if expected etag doesn't match calculated md5. + # md5 for a SwiftReader that has done nothing is + # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing + sr = self.sr('path', BytesIO(b'body'), + {'etag': md5(b'doesntmatch').hexdigest()}) + self.assertRaises(SwiftError, _consume, sr) + + sr = self.sr('path', BytesIO(b'body'), + {'etag': md5(b'body').hexdigest()}) + _consume(sr) + + # Should still work if etag was quoted + sr = self.sr('path', BytesIO(b'body'), + {'etag': '"%s"' % md5(b'body').hexdigest()}) + _consume(sr) + + # Check error is raised if SwiftReader doesn't read the same length + # as the content length it is created with + sr = self.sr('path', BytesIO(b'body'), {'content-length': 5}) + self.assertRaises(SwiftError, _consume, sr) + + sr = self.sr('path', BytesIO(b'body'), {'content-length': 4}) + _consume(sr) + + # Check that the iterator generates expected length and etag values + sr = self.sr('path', ['abc'.encode()] * 3, + {'content-length': 9, + 'etag': md5('abc'.encode() * 3).hexdigest()}) + _consume(sr) + self.assertEqual(sr._actual_read, 9) + self.assertEqual(sr._actual_md5.hexdigest(), + md5('abc'.encode() * 3).hexdigest()) + + +class _TestServiceBase(unittest.TestCase): + def _get_mock_connection(self, attempts=2): + m = Mock(spec=Connection) + type(m).attempts = PropertyMock(return_value=attempts) + type(m).auth_end_time = PropertyMock(return_value=4) + return m + + def _get_queue(self, q): + # Instead of blocking pull items straight from the queue. + # expects at least one item otherwise the test will fail. + try: + return q.get_nowait() + except QueueEmptyError: + self.fail('Expected item in queue but found none') + + def _get_expected(self, update=None): + expected = self.expected.copy() + if update: + expected.update(update) + + return expected + + +class TestServiceDelete(_TestServiceBase): + def setUp(self): + super(TestServiceDelete, self).setUp() + self.opts = {'leave_segments': False, 'yes_all': False} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be string in the form delete_XX + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'success': None # Should be a bool + } + + def test_delete_segment(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_segment', + 'object': 'test_s', + 'success': True, + }) + + r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_s', response_dict={} + ) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) + + def test_delete_segment_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.delete_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_segment', + 'object': 'test_s', + 'success': False, + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + + before = time.time() + r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + after = time.time() + + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_s', response_dict={} + ) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertIn('Traceback', r['traceback']) + + def test_delete_object(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock(return_value={}) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={}, + headers={} + ) + self.assertEqual(expected_r, r) + + def test_delete_object_with_headers(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock(return_value={}) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: Test'] + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q) + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, + query_string='symlink=get') + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={}, + headers={'Skip-Middleware': 'Test'} + ) + self.assertEqual(expected_r, r) + + def test_delete_object_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.delete_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': False, + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + # _delete_object doesn't populate attempts or response dict if it hits + # an error. This may not be the correct behaviour. + del expected_r['response_dict'], expected_r['attempts'] + + before = time.time() + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + after = time.time() + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={}, + headers={} + ) + self.assertEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertIn('Traceback', r['traceback']) + + def test_delete_object_slo_support(self): + # If SLO headers are present the delete call should include an + # additional query string to cause the right delete server side + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock( + return_value={'x-static-large-object': True} + ) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', + query_string='multipart-manifest=delete', + response_dict={}, + headers={} + ) + self.assertEqual(expected_r, r) + + def test_delete_object_dlo_support(self): + mock_q = Queue() + s = SwiftService() + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True, + 'dlo_segments_deleted': True + }) + # A DLO object is determined in _delete_object by heading the object + # and checking for the existence of a x-object-manifest header. + # Mock that here. + mock_conn.head_object = Mock( + return_value={'x-object-manifest': 'manifest_c/manifest_p'} + ) + mock_conn.get_container = Mock( + side_effect=[(None, [{'name': 'test_seg_1'}, + {'name': 'test_seg_2'}]), + (None, {})] + ) + + def get_mock_list_conn(options): + return mock_conn + + with mock.patch('swiftclient.service.get_conn', get_mock_list_conn): + r = s._delete_object( + mock_conn, 'test_c', 'test_o', self.opts, mock_q + ) + + self.assertEqual(expected_r, r) + expected = [ + mock.call('test_c', 'test_o', query_string=None, response_dict={}, + headers={}), + mock.call('manifest_c', 'test_seg_1', response_dict={}), + mock.call('manifest_c', 'test_seg_2', response_dict={})] + mock_conn.delete_object.assert_has_calls(expected, any_order=True) + + def test_delete_empty_container(self): + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': True, + 'object': None + }) + + r = SwiftService._delete_empty_container(mock_conn, 'test_c', + self.opts) + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={}, headers={} + ) + self.assertEqual(expected_r, r) + + def test_delete_empty_container_with_headers(self): + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': True, + 'object': None + }) + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: Test'] + + r = SwiftService._delete_empty_container(mock_conn, 'test_c', opt_c) + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={}, headers={'Skip-Middleware': 'Test'} + ) + self.assertEqual(expected_r, r) + + def test_delete_empty_container_exception(self): + mock_conn = self._get_mock_connection() + mock_conn.delete_container = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': False, + 'object': None, + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + + before = time.time() + s = SwiftService() + r = s._delete_empty_container(mock_conn, 'test_c', {}) + after = time.time() + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={}, headers={} + ) + self.assertEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertIn('Traceback', r['traceback']) + + @mock.patch.object(swiftclient.service.SwiftService, 'capabilities', + lambda *a: {'action': 'capabilities', + 'timestamp': time.time(), + 'success': True, + 'capabilities': { + 'bulk_delete': + {'max_deletes_per_request': 10}} + }) + def test_bulk_delete_page_size(self): + # make a list of 100 objects + obj_list = ['x%02d' % i for i in range(100)] + errors = [] + + # _bulk_delete_page_size uses 2x the number of threads to determine + # if if there are "many" object to delete or not + + # format is: [(thread_count, expected result), ...] + obj_threads_exp = [ + (10, 10), # something small + (49, 10), # just under the bounds + (50, 1), # cutover point + (51, 1), # just over bounds + (100, 1), # something big + ] + for thread_count, exp in obj_threads_exp: + s = SwiftService(options={'object_dd_threads': thread_count}) + res = s._bulk_delete_page_size(obj_list) + if res != exp: + msg = 'failed for thread_count %d: got %r expected %r' % \ + (thread_count, res, exp) + errors.append(msg) + if errors: + self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors)) + + +class TestSwiftError(unittest.TestCase): + + def test_is_exception(self): + se = SwiftError(5) + self.assertIsInstance(se, Exception) + + def test_empty_swifterror_creation(self): + se = SwiftError(5) + + self.assertEqual(se.value, 5) + self.assertIsNone(se.container) + self.assertIsNone(se.obj) + self.assertIsNone(se.segment) + self.assertIsNone(se.exception) + + self.assertEqual(str(se), '5') + + def test_swifterror_creation(self): + test_exc = Exception('test exc') + se = SwiftError(5, 'con', 'obj', 'seg', test_exc) + + self.assertEqual(se.value, 5) + self.assertEqual(se.container, 'con') + self.assertEqual(se.obj, 'obj') + self.assertEqual(se.segment, 'seg') + self.assertEqual(se.exception, test_exc) + + self.assertEqual(str(se), '5 container:con object:obj segment:seg') + + +class TestServiceUtils(unittest.TestCase): + + def setUp(self): + super(TestServiceUtils, self).setUp() + with mock.patch.dict(swiftclient.service.environ, clean_os_environ): + swiftclient.service._default_global_options = \ + swiftclient.service._build_default_global_options() + self.opts = swiftclient.service._default_global_options.copy() + + def test_process_options_defaults(self): + # The only actions that should be taken on default options set is + # to change the auth version to v2.0 and create the os_options dict + opt_c = self.opts.copy() + + swiftclient.service.process_options(opt_c) + + self.assertIn('os_options', opt_c) + del opt_c['os_options'] + self.assertEqual(opt_c['auth_version'], '2.0') + opt_c['auth_version'] = '1.0' + + self.assertEqual(opt_c, self.opts) + + def test_process_options_auth_version(self): + # auth_version should be set to 2.0 + # if it isn't already set to 3.0 + # and the v1 command line arguments aren't present + opt_c = self.opts.copy() + + # Check v3 isn't changed + opt_c['auth_version'] = '3' + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '3') + + # Check v1 isn't changed if user, key and auth are set + opt_c = self.opts.copy() + opt_c['auth_version'] = '1' + opt_c['auth'] = True + opt_c['user'] = True + opt_c['key'] = True + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '1') + + def test_process_options_new_style_args(self): + # checks new style args are copied to old style + # when old style don't exist + opt_c = self.opts.copy() + + opt_c['auth'] = '' + opt_c['user'] = '' + opt_c['key'] = '' + opt_c['os_auth_url'] = 'os_auth' + opt_c['os_username'] = 'os_user' + opt_c['os_password'] = 'os_pass' + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '2.0') + self.assertEqual(opt_c['auth'], 'os_auth') + self.assertEqual(opt_c['user'], 'os_user') + self.assertEqual(opt_c['key'], 'os_pass') + + # Check old style args are left alone if they exist + opt_c = self.opts.copy() + opt_c['auth'] = 'auth' + opt_c['user'] = 'user' + opt_c['key'] = 'key' + opt_c['os_auth_url'] = 'os_auth' + opt_c['os_username'] = 'os_user' + opt_c['os_password'] = 'os_pass' + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '1.0') + self.assertEqual(opt_c['auth'], 'auth') + self.assertEqual(opt_c['user'], 'user') + self.assertEqual(opt_c['key'], 'key') + + def test_split_headers(self): + mock_headers = ['color:blue', 'SIZE: large'] + expected = {'Color': 'blue', 'Size': 'large'} + + actual = swiftclient.service.split_headers(mock_headers) + self.assertEqual(expected, actual) + + def test_split_headers_prefix(self): + mock_headers = ['color:blue', 'size:large'] + expected = {'Prefix-Color': 'blue', 'Prefix-Size': 'large'} + + actual = swiftclient.service.split_headers(mock_headers, 'prefix-') + self.assertEqual(expected, actual) + + def test_split_headers_list_of_tuples(self): + mock_headers = [('color', 'blue'), ('size', 'large')] + expected = {'Prefix-Color': 'blue', 'Prefix-Size': 'large'} + + actual = swiftclient.service.split_headers(mock_headers, 'prefix-') + self.assertEqual(expected, actual) + + def test_split_headers_dict(self): + expected = {'Color': 'blue', 'Size': 'large'} + + actual = swiftclient.service.split_headers(expected) + self.assertEqual(expected, actual) + + def test_split_headers_error(self): + self.assertRaises(SwiftError, swiftclient.service.split_headers, + ['notvalid']) + self.assertRaises(SwiftError, swiftclient.service.split_headers, + [('also', 'not', 'valid')]) + + +class TestSwiftUploadObject(unittest.TestCase): + + def setUp(self): + self.suo = swiftclient.service.SwiftUploadObject + super(TestSwiftUploadObject, self).setUp() + + def test_create_with_string(self): + suo = self.suo('source') + self.assertEqual(suo.source, 'source') + self.assertEqual(suo.object_name, 'source') + self.assertIsNone(suo.options) + + suo = self.suo('source', 'obj_name') + self.assertEqual(suo.source, 'source') + self.assertEqual(suo.object_name, 'obj_name') + self.assertIsNone(suo.options) + + suo = self.suo('source', 'obj_name', {'opt': '123'}) + self.assertEqual(suo.source, 'source') + self.assertEqual(suo.object_name, 'obj_name') + self.assertEqual(suo.options, {'opt': '123'}) + + def test_create_with_file(self): + with tempfile.TemporaryFile() as mock_file: + # Check error is raised if no object name is provided with a + # filelike object + self.assertRaises(SwiftError, self.suo, mock_file) + + # Check that empty strings are invalid object names + self.assertRaises(SwiftError, self.suo, mock_file, '') + + suo = self.suo(mock_file, 'obj_name') + self.assertEqual(suo.source, mock_file) + self.assertEqual(suo.object_name, 'obj_name') + self.assertIsNone(suo.options) + + suo = self.suo(mock_file, 'obj_name', {'opt': '123'}) + self.assertEqual(suo.source, mock_file) + self.assertEqual(suo.object_name, 'obj_name') + self.assertEqual(suo.options, {'opt': '123'}) + + def test_create_with_no_source(self): + suo = self.suo(None, 'obj_name') + self.assertIsNone(suo.source) + self.assertEqual(suo.object_name, 'obj_name') + self.assertIsNone(suo.options) + + # Check error is raised if source is None without an object name + self.assertRaises(SwiftError, self.suo, None) + + def test_create_with_invalid_source(self): + # Source can only be None, string or filelike object, + # check an error is raised with an invalid type. + self.assertRaises(SwiftError, self.suo, []) + + +class TestServiceList(_TestServiceBase): + def setUp(self): + super(TestServiceList, self).setUp() + self.opts = {'prefix': None, 'long': False, 'delimiter': ''} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be list_X_part (account or container) + 'container': None, # Should be a string when listing a container + 'prefix': None, + 'success': None # Should be a bool + } + + def test_list_account(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c'}], + 'marker': '' + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c', 'meta': {'test_m': '1'}}], + 'marker': '', + }) + + SwiftService._list_account_job( + mock_conn, long_opts, mock_q + ) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_account_with_headers(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c'}], + 'marker': '' + }) + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: True'] + SwiftService._list_account_job( + mock_conn, opt_c, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + self.assertEqual(mock_conn.get_account.mock_calls, [ + mock.call(headers={'Skip-Middleware': 'True'}, marker='', + prefix=None), + mock.call(headers={'Skip-Middleware': 'True'}, marker='test_c', + prefix=None)]) + + def test_list_account_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_account = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': False, + 'error': self.exc, + 'marker': '', + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q) + + mock_conn.get_account.assert_called_once_with( + marker='', prefix=None, headers={} + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', long_opts, mock_q + ) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container_marker(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + + get_container_returns = [ + (None, [{'name': 'b'}, {'name': 'c'}]), + (None, []) + ] + mock_get_cont = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock_get_cont + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'b'}, {'name': 'c'}], + 'marker': 'b' + }) + + _opts = self.opts.copy() + _opts['marker'] = 'b' + SwiftService._list_container_job(mock_conn, 'test_c', _opts, mock_q) + + # This does not test if the marker is propagated, because we always + # get the final call to the get_container with the final item 'c', + # even if marker wasn't set. This test just makes sure the whole + # stack works in a sane way. + mock_kw = mock_get_cont.call_args[1] + self.assertEqual(mock_kw['marker'], 'c') + + # This tests that the lower levels get the marker delivered. + self.assertEqual(expected_r, self._get_queue(mock_q)) + + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container_with_headers(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: Test'] + + SwiftService._list_container_job( + mock_conn, 'test_c', opt_c, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c', headers={'Skip-Middleware': 'Test'}, + delimiter='', marker='', prefix=None), + mock.call('test_c', headers={'Skip-Middleware': 'Test'}, + delimiter='', marker='test_o', prefix=None)]) + + def test_list_container_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_container = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': False, + 'error': self.exc, + 'marker': '', + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + + mock_conn.get_container.assert_called_once_with( + 'test_c', marker='', delimiter='', prefix=None, headers={} + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + @mock.patch('swiftclient.service.get_conn') + def test_list_queue_size(self, mock_get_conn): + mock_conn = self._get_mock_connection() + # Return more results than should fit in the results queue + get_account_returns = [ + (None, [{'name': 'container1'}]), + (None, [{'name': 'container2'}]), + (None, [{'name': 'container3'}]), + (None, [{'name': 'container4'}]), + (None, [{'name': 'container5'}]), + (None, [{'name': 'container6'}]), + (None, [{'name': 'container7'}]), + (None, [{'name': 'container8'}]), + (None, [{'name': 'container9'}]), + (None, [{'name': 'container10'}]), + (None, [{'name': 'container11'}]), + (None, [{'name': 'container12'}]), + (None, [{'name': 'container13'}]), + (None, [{'name': 'container14'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_get_conn.return_value = mock_conn + + s = SwiftService(options=self.opts) + lg = s.list() + + # Start the generator + first_list_part = next(lg) + + # Wait for the number of calls to get_account to reach our expected + # value, then let it run some more to make sure the value remains + # stable + count = mock_conn.get_account.call_count + stable = 0 + while mock_conn.get_account.call_count != count or stable < 5: + if mock_conn.get_account.call_count == count: + stable += 1 + else: + count = mock_conn.get_account.call_count + stable = 0 + # The test requires a small sleep to allow other threads to + # execute - in this mocked environment we assume that if the call + # count to get_account has not changed in 0.25s then no more calls + # will be made. + sleep(0.05) + + stable_get_account_call_count = mock_conn.get_account.call_count + + # Collect all remaining results from the generator + list_results = [first_list_part] + list(lg) + + # Make sure the stable call count is correct - this should be 12 calls + # to get_account; + # 1 for first_list_part + # 10 for the values on the queue + # 1 for the value blocking whilst trying to place onto the queue + self.assertEqual(12, stable_get_account_call_count) + + # Make sure all the containers were listed and placed onto the queue + self.assertEqual(15, mock_conn.get_account.call_count) + + # Check the results were all returned + observed_listing = [] + for lir in list_results: + observed_listing.append( + [li['name'] for li in lir['listing']] + ) + expected_listing = [] + for gar in get_account_returns[:-1]: # The empty list is not returned + expected_listing.append( + [li['name'] for li in gar[1]] + ) + self.assertEqual(observed_listing, expected_listing) + + +class TestService(unittest.TestCase): + + def test_upload_with_bad_segment_size(self): + for bad in ('ten', '1234X', '100.3'): + options = {'segment_size': bad} + try: + service = SwiftService(options) + next(service.upload('c', 'o')) + self.fail('Expected SwiftError when segment_size=%s' % bad) + except SwiftError as exc: + self.assertEqual('Segment size should be an integer value', + exc.value) + + @mock.patch('swiftclient.service.stat') + @mock.patch('swiftclient.service.getmtime', return_value=1.0) + @mock.patch('swiftclient.service.getsize', return_value=4) + def test_upload_with_relative_path(self, *args, **kwargs): + service = SwiftService({}) + objects = [{'path': "./testobj", + 'strt_indx': 2}, + {'path': os.path.join(os.getcwd(), "testobj"), + 'strt_indx': 1}, + {'path': ".\\testobj", + 'strt_indx': 2}] + for obj in objects: + with mock.patch('swiftclient.service.Connection') as mock_conn, \ + mock.patch.object(builtins, 'open') as mock_open: + mock_open.return_value = six.StringIO('asdf') + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value =\ + 'd41d8cd98f00b204e9800998ecf8427e' + resp_iter = service.upload( + 'c', [SwiftUploadObject(obj['path'])]) + responses = [x for x in resp_iter] + for resp in responses: + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(2, len(responses)) + create_container_resp, upload_obj_resp = responses + self.assertEqual(create_container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + obj['path'][obj['strt_indx']:]) + self.assertEqual(upload_obj_resp['path'], obj['path']) + self.assertTrue(mock_open.return_value.closed) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + segment_etag = md5(b'A' * 1024).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + segment_etag + options = {'use_slo': True, 'segment_size': 1024} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertFalse('error' in resp) + self.assertTrue(resp['success']) + self.assertEqual(5, len(responses)) + container_resp, segment_container_resp = responses[0:2] + segment_response = responses[2:4] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertTrue(upload_obj_resp['large_object']) + self.assertIn('manifest_response_dict', upload_obj_resp) + self.assertEqual(upload_obj_resp['manifest_response_dict'], {}) + for i, resp in enumerate(segment_response): + self.assertEqual(i, resp['segment_index']) + self.assertEqual(1024, resp['segment_size']) + self.assertEqual('d47b127bc2de2d687ddc82dac354c415', + resp['segment_etag']) + self.assertTrue(resp['segment_location'].endswith( + '/0000000%d' % i)) + self.assertTrue(resp['segment_location'].startswith( + '/container_segments/streamed')) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream_fits_in_one_segment(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + whole_etag = md5(b'A' * 2048).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + whole_etag + options = {'use_slo': True, 'segment_size': 10240} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertNotIn('error', resp) + self.assertTrue(resp['success']) + self.assertEqual(3, len(responses)) + container_resp, segment_container_resp = responses[0:2] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertFalse(upload_obj_resp['large_object']) + self.assertNotIn('manifest_response_dict', upload_obj_resp) + + +class TestServiceUpload(_TestServiceBase): + + @contextlib.contextmanager + def assert_open_results_are_closed(self): + opened_files = [] + builtin_open = builtins.open + + def open_wrapper(*a, **kw): + opened_files.append((builtin_open(*a, **kw), a, kw)) + return opened_files[-1][0] + + with mock.patch.object(builtins, 'open', open_wrapper): + yield + for fp, args, kwargs in opened_files: + formatted_args = [repr(a) for a in args] + formatted_args.extend('%s=%r' % kv for kv in kwargs.items()) + formatted_args = ', '.join(formatted_args) + self.assertTrue(fp.closed, + 'Failed to close open(%s)' % formatted_args) + + def test_upload_object_job_file_with_unicode_path(self): + # Uploading a file results in the file object being wrapped in a + # LengthWrapper. This test sets the options in such a way that much + # of _upload_object_job is skipped bringing the critical path down + # to around 60 lines to ease testing. + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': True, + 'object': 'テスト/dummy.dat', + 'manifest_response_dict': {}, + 'segment_results': [{'action': 'upload_segment', + 'success': True}] * 3, + 'status': 'uploaded', + 'success': True, + } + expected_mtime = '%f' % os.path.getmtime(f.name) + + mock_conn = mock.Mock() + mock_conn.put_object.return_value = '' + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with mock.patch.object(s, '_upload_segment_job') as mock_job: + mock_job.return_value = { + 'action': 'upload_segment', + 'success': True} + + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='テスト/dummy.dat', + options=dict(s._options, + segment_size=10, + leave_segments=True)) + + mtime = r['headers']['x-object-meta-mtime'] + self.assertEqual(expected_mtime, mtime) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual( + 'test_c_segments/%E3%83%86%E3%82%B9%E3%83%88/dummy.dat/' + + '%s/30/10/' % mtime, r['headers']['x-object-manifest']) + del r['headers']['x-object-manifest'] + + self.assertEqual(r['path'], f.name) + del r['path'] + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with('test_c', 'テスト/dummy.dat', + '', + content_length=0, + headers={}, + response_dict={}) + + def test_upload_segment_job(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 10) + f.write(b'b' * 10) + f.write(b'c' * 10) + f.flush() + + # run read() when put_object is called to calculate md5sum + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return contents.get_md5sum() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'upload_segment', + 'for_container': 'test_c', + 'for_object': 'test_o', + 'segment_index': 2, + 'segment_size': 10, + 'segment_location': '/test_c_segments/test_s_1', + 'log_line': 'test_o segment 2', + 'success': True, + 'response_dict': {}, + 'segment_etag': md5(b'b' * 10).hexdigest(), + 'attempts': 2, + } + + s = SwiftService() + with self.assert_open_results_are_closed(): + r = s._upload_segment_job(conn=mock_conn, + path=f.name, + container='test_c', + segment_name='test_s_1', + segment_start=10, + segment_size=10, + segment_index=2, + obj_name='test_o', + options={'segment_container': None, + 'checksum': True}) + + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertIsInstance(contents, utils.LengthWrapper) + self.assertEqual(len(contents), 10) + + def test_upload_stream_segment(self): + common_params = { + 'segment_container': 'segments', + 'segment_name': 'test_stream_2', + 'container': 'test_stream', + 'object': 'stream_object', + } + tests = [ + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1024}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 512}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 512).hexdigest()}}, + # 0-sized segment should not be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 0}, + 'put_object_args': {}, + 'expected': { + 'complete': True}}, + # 0-sized objects should be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 0, + 'content_size': 0}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'').hexdigest()}}, + # Test boundary conditions + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 1023}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 1023).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 2047}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 2047).hexdigest()}}, + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1025}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + ] + + for test_args in tests: + params = test_args['test_params'] + stream = test_utils.FakeStream(params['content_size']) + segment_size = params['segment_size'] + segment_index = params['segment_index'] + + def _fake_put_object(*args, **kwargs): + contents = args[2] + # Consume and compute md5 + return md5(contents).hexdigest() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _fake_put_object + + s = SwiftService() + resp = s._upload_stream_segment( + conn=mock_conn, + container=common_params['container'], + object_name=common_params['object'], + segment_container=common_params['segment_container'], + segment_name=common_params['segment_name'], + segment_size=segment_size, + segment_index=segment_index, + headers={}, + fd=stream) + expected_args = test_args['expected'] + put_args = test_args['put_object_args'] + expected_response = { + 'segment_size': min(len(stream), segment_size), + 'complete': expected_args['complete'], + 'success': True, + } + if len(stream) or segment_index == 0: + segment_location = '/%s/%s' % (put_args['container'], + put_args['object']) + expected_response.update( + {'segment_index': segment_index, + 'segment_location': segment_location, + 'segment_etag': expected_args['segment_etag'], + 'for_object': common_params['object']}) + mock_conn.put_object.assert_called_once_with( + put_args['container'], + put_args['object'], + mock.ANY, + content_length=min(len(stream), segment_size), + headers={'etag': expected_args['segment_etag']}, + response_dict=mock.ANY) + else: + self.assertEqual([], mock_conn.put_object.mock_calls) + expected_response.update( + {'segment_index': None, + 'segment_location': None, + 'segment_etag': None}) + self.assertEqual(expected_response, resp) + + def test_etag_mismatch_with_ignore_checksum(self): + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return 'badresponseetag' + + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 10) + f.write(b'b' * 10) + f.write(b'c' * 10) + f.flush() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_segment_job(conn=mock_conn, + path=f.name, + container='test_c', + segment_name='test_s_1', + segment_start=10, + segment_size=10, + segment_index=2, + obj_name='test_o', + options={'segment_container': None, + 'checksum': False}) + + self.assertIsNone(r.get('error')) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + # Check that md5sum is not calculated. + self.assertEqual(contents.get_md5sum(), '') + + def test_upload_segment_job_etag_mismatch(self): + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return 'badresponseetag' + + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 10) + f.write(b'b' * 10) + f.write(b'c' * 10) + f.flush() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with self.assert_open_results_are_closed(): + r = s._upload_segment_job(conn=mock_conn, + path=f.name, + container='test_c', + segment_name='test_s_1', + segment_start=10, + segment_size=10, + segment_index=2, + obj_name='test_o', + options={'segment_container': None, + 'checksum': True}) + + self.assertIn('md5 mismatch', str(r.get('error'))) + + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest()) + + def test_upload_object_job_file(self): + # Uploading a file results in the file object being wrapped in a + # LengthWrapper. This test sets the options in such a way that much + # of _upload_object_job is skipped bringing the critical path down + # to around 60 lines to ease testing. + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': False, + 'object': 'test_o', + 'response_dict': {}, + 'status': 'uploaded', + 'success': True, + } + expected_mtime = '%f' % os.path.getmtime(f.name) + + # run read() when put_object is called to calculate md5sum + # md5sum is verified in _upload_object_job. + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return contents.get_md5sum() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with self.assert_open_results_are_closed(): + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options=dict(s._options, + leave_segments=True)) + + mtime = r['headers']['x-object-meta-mtime'] + self.assertEqual(expected_mtime, mtime) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual(r['path'], f.name) + del r['path'] + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with('test_c', 'test_o', + mock.ANY, + content_length=30, + headers={}, + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertIsInstance(contents, utils.LengthWrapper) + self.assertEqual(len(contents), 30) + + @mock.patch('swiftclient.service.time', return_value=1400000000) + def test_upload_object_job_stream(self, time_mock): + # Streams are wrapped as ReadableToIterable + with tempfile.TemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + f.seek(0) + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': False, + 'object': 'test_o', + 'response_dict': {}, + 'status': 'uploaded', + 'success': True, + 'path': None, + } + expected_mtime = 1400000000 + + mock_conn = mock.Mock() + mock_conn.put_object.return_value = '' + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f, + obj='test_o', + options=dict(s._options, + leave_segments=True)) + + mtime = float(r['headers']['x-object-meta-mtime']) + self.assertEqual(mtime, expected_mtime) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with('test_c', 'test_o', + mock.ANY, + content_length=None, + headers={}, + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertIsInstance(contents, utils.ReadableToIterable) + self.assertEqual(contents.chunk_size, 65536) + # next retrieves the first chunk of the stream or len(chunk_size) + # or less, it also forces the md5 to be calculated. + self.assertEqual(next(contents), b'a' * 30) + self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) + + def test_upload_object_job_etag_mismatch(self): + # The etag test for both streams and files use the same code + # so only one test should be needed. + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return 'badresponseetag' + + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options=dict(s._options, + leave_segments=True)) + + self.assertIs(r['success'], False) + self.assertIn('md5 mismatch', str(r.get('error'))) + + self.assertEqual(mock_conn.put_object.call_count, 1) + expected_headers = {'x-object-meta-mtime': mock.ANY} + mock_conn.put_object.assert_called_with('test_c', 'test_o', + mock.ANY, + content_length=30, + headers=expected_headers, + response_dict={}) + + contents = mock_conn.put_object.call_args[0][2] + self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) + + def test_upload_object_job_identical_etag(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-length': 30, + 'etag': md5(b'a' * 30).hexdigest()} + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 0}) + + self.assertIsNone(r.get('error')) + self.assertIs(True, r['success']) + self.assertEqual(r.get('status'), 'skipped-identical') + self.assertEqual(mock_conn.put_object.call_count, 0) + self.assertEqual(mock_conn.head_object.call_count, 1) + mock_conn.head_object.assert_called_with('test_c', 'test_o') + + def test_upload_object_job_identical_slo_with_nesting(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()} + mock_conn.get_object.side_effect = [ + ({}, manifest.encode('ascii')), + ({}, submanifest.encode('ascii'))] + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + + self.assertIsNone(r.get('error')) + self.assertIs(True, r['success']) + self.assertEqual('skipped-identical', r.get('status')) + self.assertEqual(0, mock_conn.put_object.call_count) + self.assertEqual([mock.call('test_c', 'test_o')], + mock_conn.head_object.mock_calls) + self.assertEqual([ + mock.call('test_c', 'test_o', + query_string='multipart-manifest=get'), + mock.call('test_c_segments', 'test_sub_slo', + query_string='multipart-manifest=get'), + ], mock_conn.get_object.mock_calls) + + def test_upload_object_job_identical_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + segment_etag = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'x-object-manifest': 'test_c_segments/test_o/prefix', + 'content-length': 30, + 'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()} + mock_conn.get_container.side_effect = [ + (None, [{"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/00"}, + {"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/01"}]), + (None, [{"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/02"}]), + (None, {})] + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + + self.assertIsNone(r.get('error')) + self.assertIs(True, r['success']) + self.assertEqual('skipped-identical', r.get('status')) + self.assertEqual(0, mock_conn.put_object.call_count) + self.assertEqual(1, mock_conn.head_object.call_count) + self.assertEqual(3, mock_conn.get_container.call_count) + mock_conn.head_object.assert_called_with('test_c', 'test_o') + expected = [ + mock.call('test_c_segments', prefix='test_o/prefix', + marker='', delimiter=None, headers={}), + mock.call('test_c_segments', prefix='test_o/prefix', + marker="test_o/prefix/01", delimiter=None, + headers={}), + mock.call('test_c_segments', prefix='test_o/prefix', + marker="test_o/prefix/02", delimiter=None, + headers={}), + ] + mock_conn.get_container.assert_has_calls(expected) + + def test_make_upload_objects(self): + check_names_pseudo_to_expected = { + (('/absolute/file/path',), ''): ['absolute/file/path'], + (('relative/file/path',), ''): ['relative/file/path'], + (('/absolute/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/absolute/file/path'], + (('/absolute/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/absolute/file/path'], + (('relative/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/relative/file/path'], + (('relative/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/relative/file/path'], + } + errors = [] + for (filenames, pseudo_folder), expected in \ + check_names_pseudo_to_expected.items(): + actual = SwiftService._make_upload_objects( + filenames, pseudo_folder=pseudo_folder) + try: + self.assertEqual(expected, [o.object_name for o in actual]) + except AssertionError as e: + msg = 'given (%r, %r) expected %r, got %s' % ( + filenames, pseudo_folder, expected, e) + errors.append(msg) + self.assertFalse(errors, "\nERRORS:\n%s" % '\n'.join(errors)) + + def test_create_dir_marker_job_unchanged(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'application/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.234000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.getmtime', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + path='test', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': 'test', + 'headers': {'x-object-meta-mtime': '1.234000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_unchanged_old_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_overwrites_bad_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/plain', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_missing(self): + mock_conn = mock.Mock() + mock_conn.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + + +class TestServiceDownload(_TestServiceBase): + + def setUp(self): + super(TestServiceDownload, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + self.opts['no_download'] = True + self.obj_content = b'c' * 10 + self.obj_etag = md5(self.obj_content).hexdigest() + self.obj_len = len(self.obj_content) + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': 'download_object', # Should always be download_object + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'success': None # Should be a bool + } + + def _readbody(self): + yield self.obj_content + + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job(self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + obj_count = [0] + + def make_counting_generator(object_to_yield, total_count): + # maintain a counter of objects yielded + count = [0] + + def counting_generator(): + while count[0] < 10: + yield object_to_yield + count[0] += 1 + total_count[0] += 1 + return counting_generator() + + obj_count_on_sub_page_call = [] + sub_page_call_count = [0] + + def fake_sub_page(*args): + # keep a record of obj_count when this function is called + obj_count_on_sub_page_call.append(obj_count[0]) + sub_page_call_count[0] += 1 + if sub_page_call_count[0] < 3: + return range(0, 10) + return None + + sub_page.side_effect = fake_sub_page + + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + as_comp.side_effect = [ + make_counting_generator(r, obj_count), + make_counting_generator(r, obj_count) + ] + + s = SwiftService() + down_gen = s._download_container('test_c', self.opts) + results = list(down_gen) + self.assertEqual(20, len(results)) + self.assertEqual(2, as_comp.call_count) + self.assertEqual(3, sub_page_call_count[0]) + self.assertEqual([0, 7, 17], obj_count_on_sub_page_call) + + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job_error( + self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + class BoomError(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + def _make_result(): + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + return r + + as_comp.side_effect = [ + + ] + # We need Futures here because the error will cause a call to .cancel() + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + BoomError('Go Boom') + ] + sub_page.side_effect = sub_page_effects + # ...but we must also mock the returns to as_completed + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)] + ] + + s = SwiftService() + self.assertRaises( + BoomError, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was an unknown error, so make sure we attempt to cancel futures + for spe in sub_page_effects[0]: + spe.cancel.assert_called_once_with() + self.assertEqual(1, as_comp.call_count) + + # Now test ClientException + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + ClientException('Go Boom') + ] + sub_page.side_effect = sub_page_effects + as_comp.reset_mock() + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)], + ] + self.assertRaises( + ClientException, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was a ClientException, so make sure we don't cancel futures + for spe in sub_page_effects[0]: + self.assertFalse(spe.cancel.called) + self.assertEqual(1, as_comp.call_count) + + def test_download_object_job(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_with_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + mock_utime.assert_called_once_with( + 'test_o', (1454113727.682512, 1454113727.682512)) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_bad_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': 'foo'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual(0, len(mock_utime.mock_calls)) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_ignore_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + _opts['ignore_mtime'] = True + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual([], mock_utime.mock_calls) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_exception(self): + mock_conn = self._get_mock_connection() + mock_conn.get_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'success': False, + 'error': self.exc, + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY + }) + + s = SwiftService() + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', self.opts) + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + + @mock.patch('swiftclient.service.interruptable_as_completed') + @mock.patch('swiftclient.service.SwiftService._download_container') + @mock.patch('swiftclient.service.SwiftService._download_object_job') + def test_download_with_objects_empty(self, mock_down_obj, + mock_down_cont, mock_as_comp): + fake_future = Future() + fake_future.set_result(1) + mock_as_comp.return_value = [fake_future] + service = SwiftService() + next(service.download('c', [], self.opts), None) + mock_down_obj.assert_not_called() + mock_down_cont.assert_not_called() + + next(service.download('c', options=self.opts), None) + self.assertTrue(mock_down_cont.called) + + def test_download_with_output_dir(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['out_directory'] = 'temp_dir' + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'temp_dir/example/test') + + def test_download_with_remove_prefix(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example/' + options['remove_prefix'] = True + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_remove_prefix_and_remove_slashes(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['remove_prefix'] = True + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir_and_remove_prefix(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['out_directory'] = 'new/dir' + options['remove_prefix'] = True + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'new/dir/test') + + def test_download_object_job_skip_identical(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + err = swiftclient.ClientException('Object GET failed', + http_status=304) + + def fake_get(*args, **kwargs): + kwargs['response_dict']['headers'] = {} + raise err + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = fake_get + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'error': err, + 'response_dict': {'headers': {}}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.get_object.call_count, 1) + mock_conn.get_object.assert_called_with( + 'test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': md5(b'a' * 30).hexdigest()}, + query_string='multipart-manifest=get', + response_dict=expected_r['response_dict']) + + def test_download_object_job_skip_identical_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + on_disk_md5 = md5(b'a' * 30).hexdigest() + segment_md5 = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.get_object.return_value = ( + {'x-object-manifest': 'test_c_segments/test_o/prefix'}, [b'']) + mock_conn.get_container.side_effect = [ + (None, [{'name': 'test_o/prefix/1', + 'bytes': 10, 'hash': segment_md5}, + {'name': 'test_o/prefix/2', + 'bytes': 10, 'hash': segment_md5}]), + (None, [{'name': 'test_o/prefix/3', + 'bytes': 10, 'hash': segment_md5}]), + (None, [])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + + err = r.pop('error') + self.assertEqual("Large object is identical", err.msg) + self.assertEqual(304, err.http_status) + + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.get_object.call_count, 1) + mock_conn.get_object.assert_called_with( + 'test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict=expected_r['response_dict']) + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/2', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/3', + headers={})]) + + def test_download_object_job_skip_identical_nested_slo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + on_disk_md5 = md5(b'a' * 30).hexdigest() + + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [manifest.encode('ascii')]), + ({'x-static-large-object': True, + 'content-length': 20, + 'etag': submanifest_etag}, + submanifest.encode('ascii'))] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + + err = r.pop('error') + self.assertEqual("Large object is identical", err.msg) + self.assertEqual(304, err.http_status) + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c_segments', + 'test_sub_slo', + query_string='multipart-manifest=get')]) + + def test_download_object_job_skip_identical_diff_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.write(b'b') + f.flush() + on_disk_md5 = md5(b'a' * 30 + b'b').hexdigest() + segment_md5 = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, + [b'']), + ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, + [b'a' * 30])] + mock_conn.get_container.side_effect = [ + (None, [{'name': 'test_o/prefix/1', + 'bytes': 10, 'hash': segment_md5}, + {'name': 'test_o/prefix/2', + 'bytes': 10, 'hash': segment_md5}]), + (None, [{'name': 'test_o/prefix/3', + 'bytes': 10, 'hash': segment_md5}]), + (None, [])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': True, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'read_length': 30, + 'attempts': 2, + 'start_time': 0, + 'headers_receipt': 1, + 'finish_time': 2, + 'auth_end_time': mock_conn.auth_end_time, + } + + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True + s = SwiftService() + with mock.patch('swiftclient.service.time', side_effect=range(3)): + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job( + conn=mock_conn, + container='test_c', + obj='test_o', + options=options) + + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/2', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/3', + headers={})]) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + response_dict={})]) + + def test_download_object_job_skip_identical_diff_nested_slo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 29) + f.flush() + on_disk_md5 = md5(b'a' * 29).hexdigest() + + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [manifest.encode('ascii')]), + ({'x-static-large-object': True, + 'content-length': 20, + 'etag': submanifest_etag}, + submanifest.encode('ascii')), + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [b'a' * 30])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': True, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'read_length': 30, + 'attempts': 2, + 'start_time': 0, + 'headers_receipt': 1, + 'finish_time': 2, + 'auth_end_time': mock_conn.auth_end_time, + } + + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True + s = SwiftService() + with mock.patch('swiftclient.service.time', side_effect=range(3)): + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job( + conn=mock_conn, + container='test_c', + obj='test_o', + options=options) + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c_segments', + 'test_sub_slo', + query_string='multipart-manifest=get'), + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + response_dict={})]) + + +class TestServicePost(_TestServiceBase): + + def setUp(self): + super(TestServicePost, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + + @mock.patch('swiftclient.service.MultiThreadingManager') + @mock.patch('swiftclient.service.ResultsIterator') + def test_object_post(self, res_iter, thread_manager): + """ + Check post method translates strings and objects to _post_object_job + calls correctly + """ + tm_instance = Mock() + thread_manager.return_value = tm_instance + + self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) + spo = swiftclient.service.SwiftPostObject( + "test_spo", + {'meta': ["meta1:test2"], "header": ["hdr1:test2"]}) + + SwiftService().post('test_c', ['test_o', spo], self.opts) + + calls = [ + mock.call( + SwiftService._post_object_job, 'test_c', 'test_o', + { + "X-Object-Meta-Meta1": "test1", + "Hdr1": "test1"}, + {}), + mock.call( + SwiftService._post_object_job, 'test_c', 'test_spo', + { + "X-Object-Meta-Meta1": "test2", + "Hdr1": "test2"}, + {}), + ] + tm_instance.object_uu_pool.submit.assert_has_calls(calls) + self.assertEqual( + tm_instance.object_uu_pool.submit.call_count, len(calls)) + + res_iter.assert_called_with( + [tm_instance.object_uu_pool.submit()] * len(calls)) + + +class TestServiceCopy(_TestServiceBase): + + def setUp(self): + super(TestServiceCopy, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + + @mock.patch('swiftclient.service.MultiThreadingManager') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_object_copy(self, inter_compl, thread_manager): + """ + Check copy method translates strings and objects to _copy_object_job + calls correctly + """ + tm_instance = Mock() + thread_manager.return_value = tm_instance + + self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) + sco = swiftclient.service.SwiftCopyObject( + "test_sco", + options={'meta': ["meta1:test2"], "header": ["hdr1:test2"], + 'destination': "/cont_new/test_sco"}) + + res = SwiftService().copy('test_c', ['test_o', sco], self.opts) + res = list(res) + + calls = [ + mock.call( + SwiftService._create_container_job, 'cont_new', headers={}), + ] + tm_instance.container_pool.submit.assert_has_calls(calls, + any_order=True) + self.assertEqual( + tm_instance.container_pool.submit.call_count, len(calls)) + + calls = [ + mock.call( + SwiftService._copy_object_job, 'test_c', 'test_o', + None, + { + "X-Object-Meta-Meta1": "test1", + "Hdr1": "test1"}, + False), + mock.call( + SwiftService._copy_object_job, 'test_c', 'test_sco', + '/cont_new/test_sco', + { + "X-Object-Meta-Meta1": "test2", + "Hdr1": "test2"}, + False), + ] + tm_instance.object_uu_pool.submit.assert_has_calls(calls) + self.assertEqual( + tm_instance.object_uu_pool.submit.call_count, len(calls)) + + inter_compl.assert_called_with( + [tm_instance.object_uu_pool.submit()] * len(calls)) + + def test_object_copy_fail_dest(self): + """ + Destination in incorrect format and destination with object + used when multiple objects are copied raises SwiftError + """ + with self.assertRaises(SwiftError): + list(SwiftService().copy('test_c', ['test_o'], + {'destination': 'cont'})) + with self.assertRaises(SwiftError): + list(SwiftService().copy('test_c', ['test_o', 'test_o2'], + {'destination': '/cont/obj'})) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py new file mode 100644 index 0000000..c972281 --- /dev/null +++ b/test/unit/test_shell.py @@ -0,0 +1,3402 @@ +# Copyright (c) 2014 Christian Schwede <christian.schwede@enovance.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import unicode_literals + +import contextlib +from genericpath import getmtime +import getpass +import hashlib +import json +import logging +import mock +import os +import tempfile +import unittest +import textwrap +from time import localtime, mktime, strftime, strptime + +import six +import sys + +import swiftclient +from swiftclient.service import SwiftError +import swiftclient.shell +import swiftclient.utils + +from os.path import basename, dirname +from .utils import ( + CaptureOutput, fake_get_auth_keystone, + FakeKeystone, StubResponse, MockHttpTest) +from swiftclient.utils import ( + EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, + SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG) + +try: + from requests.packages.urllib3.exceptions import InsecureRequestWarning +except ImportError: + InsecureRequestWarning = None + +if six.PY2: + BUILTIN_OPEN = '__builtin__.open' +else: + BUILTIN_OPEN = 'builtins.open' + +mocked_os_environ = { + 'ST_AUTH': 'http://localhost:8080/auth/v1.0', + 'ST_USER': 'test:tester', + 'ST_KEY': 'testing' +} +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' + + +def _make_args(cmd, opts, os_opts, separator='-', flags=None, cmd_args=None): + """ + Construct command line arguments for given options. + """ + args = [""] + flags = flags or [] + for k, v in opts.items(): + args.append("--" + k.replace("_", "-")) + if v is not None: + args.append(v) + for k, v in os_opts.items(): + args.append("--os" + separator + k.replace("_", separator)) + if v is not None: + args.append(v) + for flag in flags: + args.append('--%s' % flag) + if cmd: + args.append(cmd) + if cmd_args: + args.extend(cmd_args) + return args + + +def _make_env(opts, os_opts): + """ + Construct a dict of environment variables for given options. + """ + env = {} + for k, v in opts.items(): + key = 'ST_' + k.upper().replace('-', '_') + env[key] = v + for k, v in os_opts.items(): + key = 'OS_' + k.upper().replace('-', '_') + env[key] = v + return env + + +def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): + flags = flags or [] + if use_env: + # set up fake environment variables and make a minimal command line + env = _make_env(opts, os_opts) + args = _make_args(cmd, {}, {}, separator='-', flags=flags, + cmd_args=cmd_args) + else: + # set up empty environment and make full command line + env = {} + args = _make_args(cmd, opts, os_opts, separator='-', flags=flags, + cmd_args=cmd_args) + return args, env + + +@contextlib.contextmanager +def patch_disable_warnings(): + if InsecureRequestWarning is None: + # If InsecureRequestWarning isn't available, disbale_warnings won't + # be either; they both came in with + # https://github.com/requests/requests/commit/811ee4e and left again + # in https://github.com/requests/requests/commit/8e17600 + yield None + else: + with mock.patch('requests.packages.urllib3.disable_warnings') \ + as patched: + yield patched + + +@mock.patch.dict(os.environ, mocked_os_environ) +class TestShell(unittest.TestCase): + def setUp(self): + super(TestShell, self).setUp() + tmpfile = tempfile.NamedTemporaryFile(delete=False) + self.tmpfile = tmpfile.name + + def tearDown(self): + try: + os.remove(self.tmpfile) + except OSError: + pass + super(TestShell, self).tearDown() + + @mock.patch('swiftclient.service.Connection') + def test_stat_account(self, connection): + argv = ["", "stat"] + return_headers = { + 'x-account-container-count': '1', + 'x-account-object-count': '2', + 'x-account-bytes-used': '3', + 'content-length': 0, + 'date': ''} + connection.return_value.head_account.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Containers: 1\n' + ' Objects: 2\n' + ' Bytes: 3\n') + + @mock.patch('swiftclient.service.Connection') + def test_stat_account_with_headers(self, connection): + argv = ["", "stat", "-H", "Skip-Middleware: Test"] + return_headers = { + 'x-account-container-count': '1', + 'x-account-object-count': '2', + 'x-account-bytes-used': '3', + 'content-length': 0, + 'date': ''} + connection.return_value.head_account.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Containers: 1\n' + ' Objects: 2\n' + ' Bytes: 3\n') + self.assertEqual(connection.return_value.head_account.mock_calls, [ + mock.call(headers={'Skip-Middleware': 'Test'})]) + + @mock.patch('swiftclient.service.Connection') + def test_stat_container(self, connection): + return_headers = { + 'x-container-object-count': '1', + 'x-container-bytes-used': '2', + 'x-container-read': 'test2:tester2', + 'x-container-write': 'test3:tester3', + 'x-container-sync-to': 'other', + 'x-container-sync-key': 'secret', + } + argv = ["", "stat", "container"] + connection.return_value.head_container.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Container: container\n' + ' Objects: 1\n' + ' Bytes: 2\n' + ' Read ACL: test2:tester2\n' + 'Write ACL: test3:tester3\n' + ' Sync To: other\n' + ' Sync Key: secret\n') + + @mock.patch('swiftclient.service.Connection') + def test_stat_container_with_headers(self, connection): + return_headers = { + 'x-container-object-count': '1', + 'x-container-bytes-used': '2', + 'x-container-read': 'test2:tester2', + 'x-container-write': 'test3:tester3', + 'x-container-sync-to': 'other', + 'x-container-sync-key': 'secret', + } + argv = ["", "stat", "container", "-H", "Skip-Middleware: Test"] + connection.return_value.head_container.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Container: container\n' + ' Objects: 1\n' + ' Bytes: 2\n' + ' Read ACL: test2:tester2\n' + 'Write ACL: test3:tester3\n' + ' Sync To: other\n' + ' Sync Key: secret\n') + self.assertEqual(connection.return_value.head_container.mock_calls, [ + mock.call('container', headers={'Skip-Middleware': 'Test'})]) + + @mock.patch('swiftclient.service.Connection') + def test_stat_object(self, connection): + return_headers = { + 'x-object-manifest': 'manifest', + 'etag': 'md5', + 'last-modified': 'yesterday', + 'content-type': 'text/plain', + 'content-length': 42, + } + argv = ["", "stat", "container", "object"] + connection.return_value.head_object.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + ' Container: container\n' + ' Object: object\n' + ' Content Type: text/plain\n' + 'Content Length: 42\n' + ' Last Modified: yesterday\n' + ' ETag: md5\n' + ' Manifest: manifest\n') + + @mock.patch('swiftclient.service.Connection') + def test_stat_object_with_headers(self, connection): + return_headers = { + 'x-object-manifest': 'manifest', + 'etag': 'md5', + 'last-modified': 'yesterday', + 'content-type': 'text/plain', + 'content-length': 42, + } + argv = ["", "stat", "container", "object", + "-H", "Skip-Middleware: Test"] + connection.return_value.head_object.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + ' Container: container\n' + ' Object: object\n' + ' Content Type: text/plain\n' + 'Content Length: 42\n' + ' Last Modified: yesterday\n' + ' ETag: md5\n' + ' Manifest: manifest\n') + self.assertEqual(connection.return_value.head_object.mock_calls, [ + mock.call('container', 'object', + headers={'Skip-Middleware': 'Test'})]) + + @mock.patch('swiftclient.service.Connection') + def test_list_json(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], + [None, []], + ] + + argv = ["", "list", "--json"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + listing = [{'name': 'container'}, + {'name': u'\u263A', 'some-custom-key': 'and value'}] + expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' + self.assertEqual(output.out, expected) + + @mock.patch('swiftclient.service.Connection') + def test_list_account(self, connection): + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, []], + ] + + argv = ["", "list"] + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, 'container\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_account_with_headers(self, connection): + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, []], + ] + + argv = ["", "list", '-H', 'Skip-Custom-Middleware: True'] + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, + headers={'Skip-Custom-Middleware': 'True'}), + mock.call(marker='container', prefix=None, + headers={'Skip-Custom-Middleware': 'True'})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, 'container\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_account_long(self, connection): + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], + [None, []], + ] + + argv = ["", "list", "--lh"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, + ' 0 0 1970-01-01 00:00:01 container\n' + ' 0 0\n') + + # Now test again, this time without returning metadata + connection.return_value.head_container.return_value = {} + + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], + [None, []], + ] + + argv = ["", "list", "--lh"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, + ' 0 0 ????-??-?? ??:??:?? container\n' + ' 0 0\n') + + def test_list_account_totals_error(self): + # No --lh provided: expect info message about incorrect --totals use + argv = ["", "list", "--totals"] + + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(output.err, + "Listing totals only works with -l or --lh.\n") + + @mock.patch('swiftclient.service.Connection') + def test_list_account_totals(self, connection): + + # Test account listing, only total count and size + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container1', 'bytes': 1, 'count': 2}, + {'name': 'container2', 'bytes': 2, 'count': 4}]], + [None, []], + ] + + argv = ["", "list", "--lh", "--totals"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + self.assertEqual(output.out, ' 6 3\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_container(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object_a'}]], + [None, []], + ] + argv = ["", "list", "container"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', + delimiter=None, prefix=None, headers={}), + mock.call('container', marker='object_a', + delimiter=None, prefix=None, headers={})] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEqual(output.out, 'object_a\n') + + # Test container listing with --long + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object_a', 'bytes': 0, + 'content_type': 'type/content', + 'last_modified': '123T456'}]], + [None, []], + ] + argv = ["", "list", "container", "--long"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', + delimiter=None, prefix=None, headers={}), + mock.call('container', marker='object_a', + delimiter=None, prefix=None, headers={})] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEqual(output.out, + ' 0 123 456' + ' type/content object_a\n' + ' 0\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_container_with_headers(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object_a'}]], + [None, []], + ] + argv = ["", "list", "container", "-H", "Skip-Middleware: Test"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', + delimiter=None, prefix=None, + headers={'Skip-Middleware': 'Test'}), + mock.call('container', marker='object_a', + delimiter=None, prefix=None, + headers={'Skip-Middleware': 'Test'})] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEqual(output.out, 'object_a\n') + + @mock.patch('swiftclient.service.makedirs') + @mock.patch('swiftclient.service.Connection') + def test_download(self, connection, makedirs): + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent), + ({'content-type': 'text/plain', + 'etag': EMPTY_ETAG}, + '') + ] + + # Test downloading whole container + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, [{'name': 'pseudo/'}]], + [None, []], + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "container"] + swiftclient.shell.main(argv) + calls = [mock.call('container', 'object', + headers={}, resp_chunk_size=65536, + response_dict={}), + mock.call('container', 'pseudo/', + headers={}, resp_chunk_size=65536, + response_dict={})] + connection.return_value.get_object.assert_has_calls( + calls, any_order=True) + mock_open.assert_called_once_with('object', 'wb', 65536) + self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) + makedirs.reset_mock() + + # Test downloading single object + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "container", "object"] + swiftclient.shell.main(argv) + connection.return_value.get_object.assert_called_with( + 'container', 'object', headers={}, resp_chunk_size=65536, + response_dict={}) + mock_open.assert_called_with('object', 'wb', 65536) + self.assertEqual([], makedirs.mock_calls) + + # Test downloading without md5 checks + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + with mock.patch(BUILTIN_OPEN) as mock_open, mock.patch( + 'swiftclient.service._SwiftReader') as sr: + argv = ["", "download", "container", "object", "--ignore-check"] + swiftclient.shell.main(argv) + connection.return_value.get_object.assert_called_with( + 'container', 'object', headers={}, resp_chunk_size=65536, + response_dict={}) + mock_open.assert_called_with('object', 'wb', 65536) + sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) + self.assertEqual([], makedirs.mock_calls) + + # Test downloading single object to stdout + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + with CaptureOutput() as output: + argv = ["", "download", "--output", "-", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual('objcontent', output.out) + + @mock.patch('swiftclient.service.shuffle') + @mock.patch('swiftclient.service.Connection') + def test_download_shuffle(self, connection, mock_shuffle): + # Test that the container and object lists are shuffled + mock_shuffle.side_effect = lambda l: l + connection.return_value.get_object.return_value = [ + {'content-type': 'text/plain', + 'etag': EMPTY_ETAG}, + ''] + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "--all"] + swiftclient.shell.main(argv) + self.assertEqual(3, mock_shuffle.call_count) + mock_shuffle.assert_any_call(['container']) + mock_shuffle.assert_any_call(['object']) + mock_shuffle.assert_any_call(['pseudo/']) + mock_open.assert_called_once_with('container/object', 'wb', 65536) + self.assertEqual([ + mock.call('container'), + mock.call('container/pseudo'), + ], mock_mkdir.mock_calls) + + # Test that the container and object lists are not shuffled + mock_shuffle.reset_mock() + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "--all", "--no-shuffle"] + swiftclient.shell.main(argv) + self.assertEqual(0, mock_shuffle.call_count) + mock_open.assert_called_once_with('container/object', 'wb', 65536) + self.assertEqual([ + mock.call('container'), + mock.call('container/pseudo'), + ], mock_mkdir.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_download_no_content_type(self, connection): + connection.return_value.get_object.return_value = [ + {'etag': EMPTY_ETAG}, + ''] + + # Test downloading whole container + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, [{'name': 'pseudo/'}]], + [None, []], + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + + with mock.patch(BUILTIN_OPEN) as mock_open: + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "container"] + swiftclient.shell.main(argv) + calls = [mock.call('container', 'object', + headers={}, resp_chunk_size=65536, + response_dict={}), + mock.call('container', 'pseudo/', + headers={}, resp_chunk_size=65536, + response_dict={})] + connection.return_value.get_object.assert_has_calls( + calls, any_order=True) + mock_open.assert_called_once_with('object', 'wb', 65536) + self.assertEqual([ + mock.call('pseudo'), + ], mock_mkdir.mock_calls) + + @mock.patch('swiftclient.shell.walk') + @mock.patch('swiftclient.service.Connection') + def test_upload(self, connection, walk): + connection.return_value.head_object.return_value = { + 'content-length': '0'} + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "-H", "X-Storage-Policy:one", + "--meta", "Color:Blue"] + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_called_once_with( + 'container', + {'X-Storage-Policy': 'one'}, + response_dict={}) + + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY, + 'X-Storage-Policy': 'one', + 'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + + # upload to pseudo-folder (via <container> param) + argv = ["", "upload", "container/pseudo-folder/nested", self.tmpfile, + "-H", "X-Storage-Policy:one"] + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_called_with( + 'container', + {'X-Storage-Policy': 'one'}, + response_dict={}) + + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY, + 'X-Storage-Policy': 'one'}, + response_dict={}) + + # Upload whole directory + argv = ["", "upload", "container", "/tmp"] + _tmpfile = self.tmpfile + _tmpfile_dir = dirname(_tmpfile) + _tmpfile_base = basename(_tmpfile) + walk.return_value = [(_tmpfile_dir, [], [_tmpfile_base])] + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + + # Upload in segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container", self.tmpfile, "-S", "10"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {'X-Storage-Policy': mock.ANY}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': mock.ANY}, + response_dict={})] + connection.return_value.put_container.has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + '', + content_length=0, + headers={'x-object-manifest': mock.ANY, + 'x-object-meta-mtime': mock.ANY}, + response_dict={}) + + # upload in segments to pseudo-folder (via <container> param) + connection.reset_mock() + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container/pseudo-folder/nested", + self.tmpfile, "-S", "10", "--use-slo"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': 'one'}, + response_dict={})] + connection.return_value.put_container.assert_has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + headers={ + 'x-object-meta-mtime': mock.ANY, + }, + query_string='multipart-manifest=put', + response_dict=mock.ANY) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_object_with_account_readonly(self, upload): + argv = ["", "upload", "container", self.tmpfile] + upload.return_value = [ + {"success": False, + "headers": {}, + "container": 'container', + "action": 'create_container', + "error": swiftclient.ClientException( + 'Container PUT failed', + http_status=403, + http_reason='Forbidden', + http_response_content=b'<html><h1>Forbidden</h1>') + }] + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + self.assertTrue(output.err != '') + warning_msg = "Warning: failed to create container 'container': " \ + "403 Forbidden" + self.assertTrue(output.err.startswith(warning_msg)) + + @mock.patch('swiftclient.service.Connection') + def test_upload_delete_slo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '2'}, + {'x-static-large-object': 'false', # For the 1st delete call + 'content-length': '2'}, + {'x-static-large-object': 'false', # For the 2nd delete call + 'content-length': '2'} + ] + connection.return_value.get_object.return_value = ( + {}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) + connection.return_value.put_object.return_value = EMPTY_ETAG + # create the delete_object child mock here in attempt to fix + # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + expected_delete_calls = [ + mock.call( + 'container1', 'old_seg1', + response_dict={} + ), + mock.call( + 'container2', 'old_seg2', + response_dict={} + ) + ] + self.assertEqual( + sorted(expected_delete_calls), + sorted(connection.return_value.delete_object.mock_calls) + ) + + @mock.patch('swiftclient.service.Connection') + def test_upload_over_symlink_to_slo(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', + 'content-location': '/v1/a/c/manifest', + 'content-length': '2'}, + ] + connection.return_value.get_object.return_value = ( + {'content-location': '/v1/a/c/manifest'}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.delete_object.return_value = None + argv = ["", "upload", "container", self.tmpfile] + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertEqual([], connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_upload_leave_slo_segments(self, connection): + # Test upload overwriting a manifest respects --leave-segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '2'}] + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_reupload_leaves_slo_segments(self, connection): + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + mtime = '{:.6f}'.format(os.path.getmtime(self.tmpfile)) + expected_segments = [ + 'container_segments/{}/slo/{}/20/10/{:08d}'.format( + self.tmpfile[1:], mtime, i) + for i in range(2) + ] + + # Test re-upload overwriting a manifest doesn't remove + # segments it just wrote + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "--use-slo", "-S", "10"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '20'}] + connection.return_value.get_object.return_value = ( + {}, + # we've already *got* the expected manifest! + json.dumps([ + {'name': seg} for seg in expected_segments + ]).encode('ascii') + ) + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile[1:], # drop leading / + mock.ANY, + headers={'x-object-meta-mtime': mtime}, + query_string='multipart-manifest=put', + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_upload_delete_dlo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile] + connection.return_value.head_object.side_effect = [ + {'x-object-manifest': 'container1/prefix', + 'content-length': '0'}, + {}, + {} + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'prefix_a', 'bytes': 0, + 'last_modified': '123T456'}]], + # Have multiple pages worth of DLO segments + [None, [{'name': 'prefix_b', 'bytes': 0, + 'last_modified': '123T456'}]], + [None, []] + ] + connection.return_value.put_object.return_value = EMPTY_ETAG + # create the delete_object child mock here in attempt to fix + # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + expected_delete_calls = [ + mock.call( + 'container1', 'prefix_a', + response_dict={} + ), + mock.call( + 'container1', 'prefix_b', + response_dict={} + ) + ] + self.assertEqual( + sorted(expected_delete_calls), + sorted(connection.return_value.delete_object.mock_calls) + ) + + @mock.patch('swiftclient.service.Connection') + def test_upload_leave_dlo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] + connection.return_value.head_object.side_effect = [ + {'x-object-manifest': 'container1/prefix', + 'content-length': '0'}] + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_upload_segments_to_same_container(self, connection): + # Upload in segments to same container + connection.return_value.head_object.return_value = { + 'content-length': '0'} + connection.return_value.attempts = 0 + connection.return_value.put_object.return_value = EMPTY_ETAG + argv = ["", "upload", "container", self.tmpfile, "-S", "10", + "-C", "container"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_called_once_with( + 'container', {}, response_dict={}) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + '', + content_length=0, + headers={'x-object-manifest': mock.ANY, + 'x-object-meta-mtime': mock.ANY}, + response_dict={}) + + @mock.patch('swiftclient.shell.io.open') + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin(self, upload_mock, io_open_mock): + def fake_open(fd, mode): + mock_io = mock.Mock() + mock_io.fileno.return_value = fd + return mock_io + + io_open_mock.side_effect = fake_open + + argv = ["", "upload", "container", "-", "--object-name", "foo"] + swiftclient.shell.main(argv) + upload_mock.assert_called_once_with("container", mock.ANY) + # This is a little convoluted: we want to examine the first call ([0]), + # the argv list([1]), the second parameter ([1]), and the first + # element. This is because the upload method takes a container and a + # list of SwiftUploadObjects. + swift_upload_obj = upload_mock.mock_calls[0][1][1][0] + self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) + io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_no_name(self, upload_mock): + argv = ["", "upload", "container", "-"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find('object-name must be specified') >= 0) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_and_others(self, upload_mock): + argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find( + 'upload from stdin cannot be used') >= 0) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 0) + @mock.patch('swiftclient.service.Connection') + def test_delete_bad_threads(self, mock_connection): + mock_connection.return_value.get_container.return_value = (None, []) + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_account(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all"] + connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.delete_object.assert_has_calls([ + mock.call('container', 'object', query_string=None, + response_dict={}, headers={}), + mock.call('container', 'obj\xe9ct2', query_string=None, + response_dict={}, headers={}), + mock.call('container2', 'object', query_string=None, + response_dict={}, headers={})], any_order=True) + self.assertEqual(3, connection.return_value.delete_object.call_count, + 'Expected 3 calls but found\n%r' + % connection.return_value.delete_object.mock_calls) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'object3'}]], + [None, []], + [None, [{'name': 'object'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "2"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + self.assertEqual( + 3, len(connection.return_value.post_account.mock_calls), + 'Expected 3 calls but found\n%r' + % connection.return_value.post_account.mock_calls) + # POSTs for same container are made in parallel so expect any order + for expected in [ + mock.call(query_string='bulk-delete', + data=b'/container/object\n/container/obj%C3%A9ct2\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b'/container/object3\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]: + self.assertIn(expected, + connection.return_value.post_account.mock_calls[:2]) + # POSTs for different containers are made sequentially so expect order + self.assertEqual( + mock.call(query_string='bulk-delete', + data=b'/container2/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + connection.return_value.post_account.mock_calls[2]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account_with_capabilities(self, connection): + connection.return_value.get_capabilities.return_value = { + 'bulk_delete': { + 'max_deletes_per_request': 10000, + 'max_failed_deletes': 1000, + }, + } + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "1"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + self.assertEqual( + connection.return_value.post_account.mock_calls, [ + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/object\n', + b'/container/obj%C3%A9ct2\n', + b'/container/z_object\n', + b'/container/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/object\n', + b'/container2/obj%C3%A9ct2\n', + b'/container2/z_object\n', + b'/container2/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + self.assertEqual(connection.return_value.get_capabilities.mock_calls, + [mock.call(None)]) # only one /info request + + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account_with_capabilities_and_pages(self, connection): + connection.return_value.get_capabilities.return_value = { + 'bulk_delete': { + 'max_deletes_per_request': 2, + 'max_failed_deletes': 1000, + }, + } + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "1"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + # check that each bulk call was only called with 2 objects + self.assertEqual( + connection.return_value.post_account.mock_calls, [ + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/object\n', + b'/container/obj%C3%A9ct2\n', + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/z_object\n', + b'/container/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/object\n', + b'/container2/obj%C3%A9ct2\n', + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/z_object\n', + b'/container2/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + self.assertEqual(connection.return_value.get_capabilities.mock_calls, + [mock.call(None)]) # only one /info request + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_container(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "container"] + connection.return_value.head_object.return_value = {} + swiftclient.shell.main(argv) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, headers={}) + connection.return_value.delete_object.assert_called_with( + 'container', 'object', query_string=None, response_dict={}, + headers={}) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_container_headers(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "container", "-H", "Skip-Middleware: Test"] + connection.return_value.head_object.return_value = {} + swiftclient.shell.main(argv) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, + headers={'Skip-Middleware': 'Test'}) + connection.return_value.delete_object.assert_called_with( + 'container', 'object', query_string=None, response_dict={}, + headers={'Skip-Middleware': 'Test'}) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_container(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "container"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, headers={}) + + def test_delete_verbose_output_utf8(self): + container = 't\u00e9st_c' + base_argv = ['', '--verbose', 'delete'] + + # simulate container having an object with utf-8 code points in name, + # just returning the object delete result + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_object', + 'object': 'obj_t\u00east_o'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 'obj_t\u00east_o [after 2 attempts]') >= 0, out) + + # simulate empty container + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_container'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 't\u00e9st_c [after 2 attempts]') >= 0, out) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_per_object(self, connection): + argv = ["", "delete", "container", "object"] + connection.return_value.head_object.return_value = {} + connection.return_value.attempts = 0 + swiftclient.shell.main(argv) + connection.return_value.delete_object.assert_called_with( + 'container', 'object', query_string=None, response_dict={}, + headers={}) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_object(self, connection): + argv = ["", "delete", "container", "object"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + connection.return_value.attempts = 0 + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + + def test_delete_verbose_output(self): + del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': 't\xe9st_c', 'action': 'delete_object', + 'object': 't\xe9st_o'} + + del_seg_res = del_obj_res.copy() + del_seg_res.update({'action': 'delete_segment'}) + + del_con_res = del_obj_res.copy() + del_con_res.update({'action': 'delete_container', 'object': None}) + + test_exc = Exception('t\xe9st_exc') + error_res = del_obj_res.copy() + error_res.update({'success': False, 'error': test_exc, 'object': None}) + + mock_delete = mock.Mock() + base_argv = ['', '--verbose', 'delete'] + + with mock.patch('swiftclient.shell.SwiftService.delete', mock_delete): + with CaptureOutput() as out: + mock_delete.return_value = [del_obj_res] + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) + + mock_delete.assert_called_once_with(container='t\xe9st_c', + objects=['t\xe9st_o']) + self.assertTrue(out.out.find( + 't\xe9st_o [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [del_seg_res] + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) + + mock_delete.assert_called_with(container='t\xe9st_c', + objects=['t\xe9st_o']) + self.assertTrue(out.out.find( + 't\xe9st_c/t\xe9st_o [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [del_con_res] + swiftclient.shell.main(base_argv + ['t\xe9st_c']) + + mock_delete.assert_called_with(container='t\xe9st_c') + self.assertTrue(out.out.find( + 't\xe9st_c [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [error_res] + self.assertRaises(SystemExit, + swiftclient.shell.main, + base_argv + ['t\xe9st_c']) + + mock_delete.assert_called_with(container='t\xe9st_c') + self.assertTrue(out.err.find( + 'Error Deleting: t\xe9st_c: t\xe9st_exc') >= 0) + + @mock.patch('swiftclient.service.Connection') + def test_post_account(self, connection): + argv = ["", "post"] + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + headers={}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_account_bad_auth(self, connection): + argv = ["", "post"] + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={'X-Trans-Id': 'trans_id'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: trans_id\n') + + # do it again with a unicode token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': 'non\u2011utf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\n' + 'Failed Transaction ID: non\u2011utf8\n') + + # do it again with a wonky token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': b'non\xffutf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: non%FFutf8\n') + + @mock.patch('swiftclient.service.Connection') + def test_post_account_not_found(self, connection): + argv = ["", "post"] + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException('test', http_status=404) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'Account not found\n') + + @mock.patch('swiftclient.service.Connection') + def test_post_container(self, connection): + argv = ["", "post", "container"] + swiftclient.shell.main(argv) + connection.return_value.post_container.assert_called_with( + 'container', headers={}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_container_bad_auth(self, connection): + argv = ["", "post", "container"] + connection.return_value.post_container.side_effect = \ + swiftclient.ClientException('bad auth') + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'bad auth\n') + + @mock.patch('swiftclient.service.Connection') + def test_post_container_not_found_causes_put(self, connection): + argv = ["", "post", "container"] + connection.return_value.post_container.side_effect = \ + swiftclient.ClientException('test', http_status=404) + swiftclient.shell.main(argv) + self.assertEqual('container', + connection.return_value.put_container.call_args[0][0]) + + def test_post_container_with_bad_name(self): + argv = ["", "post", "conta/iner"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('WARNING: / in')) + + @mock.patch('swiftclient.service.Connection') + def test_post_container_with_options(self, connection): + argv = ["", "post", "container", + "--read-acl", "test2:tester2", + "--write-acl", "test3:tester3 test4", + "--sync-to", "othersite", + "--sync-key", "secret", + ] + swiftclient.shell.main(argv) + connection.return_value.post_container.assert_called_with( + 'container', headers={ + 'X-Container-Write': 'test3:tester3 test4', + 'X-Container-Read': 'test2:tester2', + 'X-Container-Sync-Key': 'secret', + 'X-Container-Sync-To': 'othersite'}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_object(self, connection): + argv = ["", "post", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain" + ] + swiftclient.shell.main(argv) + connection.return_value.post_object.assert_called_with( + 'container', 'object', headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_object_bad_auth(self, connection): + argv = ["", "post", "container", "object"] + connection.return_value.post_object.side_effect = \ + swiftclient.ClientException("bad auth") + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'bad auth\n') + + def test_post_object_too_many_args(self): + argv = ["", "post", "container", "object", "bad_arg"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('Usage')) + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_no_destination(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain" + ] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination=None, fresh_metadata=False, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + self.assertEqual(output.out, 'container/object copied to <self>\n') + + @mock.patch('swiftclient.service.Connection') + def test_copy_object(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain", + "--destination", "/c/o" + ] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination="/c/o", + fresh_metadata=False, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + self.assertEqual( + output.out, + 'created container c\ncontainer/object copied to /c/o\n' + ) + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_fresh_metadata(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", "--fresh-metadata", + "--header", "content-type:text/plain", + "--destination", "/c/o" + ] + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination="/c/o", fresh_metadata=True, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue"] + connection.return_value.copy_object.return_value = None + swiftclient.shell.main(argv) + calls = [ + mock.call( + 'container', 'object', destination=None, + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}), + mock.call( + 'container', 'object2', destination=None, + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + ] + connection.return_value.copy_object.assert_has_calls( + calls, any_order=True) + self.assertEqual(len(connection.return_value.copy_object.mock_calls), + len(calls)) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects_destination(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue", "--destination", "/c"] + connection.return_value.copy_object.return_value = None + swiftclient.shell.main(argv) + calls = [ + mock.call( + 'container', 'object', destination="/c/object", + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}), + mock.call( + 'container', 'object2', destination="/c/object2", + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + ] + connection.return_value.copy_object.assert_has_calls( + calls, any_order=True) + self.assertEqual(len(connection.return_value.copy_object.mock_calls), + len(calls)) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects_bad_destination(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue", "--destination", "/c/o"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual( + output.err, + 'Combination of multiple objects and destination ' + 'including object is invalid\n') + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_bad_auth(self, connection): + argv = ["", "copy", "container", "object"] + connection.return_value.copy_object.side_effect = \ + swiftclient.ClientException("bad auth") + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'bad auth\n') + + def test_copy_object_not_enough_args(self): + argv = ["", "copy", "container"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('Usage')) + + def test_copy_bad_container(self): + argv = ["", "copy", "cont/ainer", "object"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('WARN')) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_prefix_based(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", + "secret_key", "--prefix-based"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=True, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_iso8601_in(self, temp_url): + dates = ('1970-01-01T00:01:00Z', '1970-01-01T00:01:00', + '1970-01-01') + for d in dates: + argv = ["", "tempurl", "GET", d, "/v1/AUTH_account/c/", + "secret_key"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_iso8601_out(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", + "secret_key", "--iso8601"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, + iso8601=True, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_absolute_expiry_temp_url(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--absolute"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_with_ip_range(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--ip-range", "1.2.3.4"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range='1.2.3.4') + + def test_temp_url_output(self): + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "http://saio:8080/v1/a/c/o", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + expected = "http://saio:8080%s" % expected + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/", + "secret_key", "--absolute", "--prefix"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' + expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_prefix=\n" % sig) + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/", + "secret_key", "--absolute", "--prefix", '--iso8601'] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' + expires = '1970-01-01T00:01:00Z' + expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=%s" + "&temp_url_prefix=\n" % (sig, expires)) + self.assertEqual(expected, output.out) + + dates = ("1970-01-01T00:01:00Z", + strftime(EXPIRES_ISO8601_FORMAT[:-1], localtime(60))) + for d in dates: + argv = ["", "tempurl", "GET", d, "/v1/a/c/o", + "secret_key"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + + ts = str(int( + mktime(strptime('2005-05-01', SHORT_EXPIRES_ISO8601_FORMAT)))) + + argv = ["", "tempurl", "GET", ts, "/v1/a/c/", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + expected = output.out + + argv = ["", "tempurl", "GET", '2005-05-01', "/v1/a/c/", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" + expected = ( + "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_ip_range=1.2.3.4\n" % sig + ) + self.assertEqual(expected, output.out) + + def test_temp_url_error_output(self): + expected = 'path must be full path to an object e.g. /v1/a/c/o\n' + for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', + 'http://saio/v1/a/c', 'http://v1/a/c/o'): + argv = ["", "tempurl", "GET", "60", bad_path, + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for path %r' % + (expected, output.err, bad_path)) + + expected = 'path must at least contain /v1/a/c/\n' + argv = ["", "tempurl", "GET", "60", '/v1/a/c', + "secret_key", "--absolute", '--prefix-based'] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for path %r' % + (expected, output.err, '/v1/a/c')) + + expected = TIME_ERRMSG + '\n' + for bad_time in ('not_an_int', '-1', '2015-05', '2015-05-01T01:00'): + argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for time %r' % + (expected, output.err, bad_time)) + + @mock.patch('swiftclient.service.Connection') + def test_capabilities(self, connection): + argv = ["", "capabilities"] + connection.return_value.get_capabilities.return_value = {'swift': None} + swiftclient.shell.main(argv) + connection.return_value.get_capabilities.assert_called_with(None) + + @mock.patch('swiftclient.service.Connection') + def test_capabilities_json(self, connection): + capabilities = { + 'slo': {'min_segment_size': 1000000}, + 'some': [{'arbitrary': 'nested'}, {'crazy': 'structure'}], + 'swift': {'version': '2.5.0'}} + + connection.return_value.get_capabilities.return_value = capabilities + argv = ["", "capabilities", "--json"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + expected = json.dumps(capabilities, sort_keys=True, indent=2) + '\n' + self.assertEqual(expected, output.out) + connection.return_value.get_capabilities.assert_called_with(None) + + def test_human_readable_upload_segment_size(self): + def _check_expected(x, expected): + actual = x.call_args_list[-1][1]["options"]["segment_size"] + self.assertEqual(int(actual), expected) + + mock_swift = mock.MagicMock(spec=swiftclient.shell.SwiftService) + with mock.patch("swiftclient.shell.SwiftService", mock_swift): + with CaptureOutput(suppress_systemexit=True) as output: + # Test new behaviour with both upper and lower case + # trailing characters + argv = ["", "upload", "-S", "1B", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1) + + argv = ["", "upload", "-S", "1K", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1024) + + argv = ["", "upload", "-S", "1m", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1048576) + + argv = ["", "upload", "-S", "1G", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1073741824) + + # Test old behaviour is not affected + argv = ["", "upload", "-S", "12345", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 12345) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + # Test invalid states + argv = ["", "upload", "-S", "1234X", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "Invalid segment size\n") + output.clear() + + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S", "K1234", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "Invalid segment size\n") + output.clear() + + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S", "K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "Invalid segment size\n") + + def test_negative_upload_segment_size(self): + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S", "-40", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40M", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40G", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + + +class TestSubcommandHelp(unittest.TestCase): + + def test_subcommand_help(self): + for command in swiftclient.shell.commands: + help_var = 'st_%s_help' % command + options_var = 'st_%s_options' % command + self.assertTrue(hasattr(swiftclient.shell, help_var)) + with CaptureOutput() as out: + argv = ['', command, '--help'] + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + expected = 'Usage: swift %s %s\n%s' % ( + command, vars(swiftclient.shell).get(options_var, "\n"), + vars(swiftclient.shell)[help_var]) + self.assertEqual(out.strip('\n'), expected) + + def test_no_help(self): + with CaptureOutput() as out: + argv = ['', 'bad_command', '--help'] + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + expected = 'no such command: bad_command' + self.assertEqual(out.strip('\n'), expected) + + +@mock.patch.dict(os.environ, mocked_os_environ) +class TestDebugAndInfoOptions(unittest.TestCase): + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_option_after_posarg(self, connection, mock_logging): + argv = ["", "stat", "--info"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.INFO) + + argv = ["", "stat", "--debug"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.DEBUG) + + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_debug_trumps_info(self, connection, mock_logging): + argv_scenarios = (["", "stat", "--info", "--debug"], + ["", "stat", "--debug", "--info"], + ["", "--info", "stat", "--debug"], + ["", "--debug", "stat", "--info"], + ["", "--info", "--debug", "stat"], + ["", "--debug", "--info", "stat"]) + for argv in argv_scenarios: + mock_logging.reset_mock() + swiftclient.shell.main(argv) + try: + mock_logging.assert_called_once_with(level=logging.DEBUG) + except AssertionError: + self.fail('Unexpected call(s) %r for args %r' + % (mock_logging.call_args_list, argv)) + + +class TestBase(unittest.TestCase): + """ + Provide some common methods to subclasses + """ + def _remove_swift_env_vars(self): + self._environ_vars = {} + keys = list(os.environ.keys()) + for k in keys: + if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') or + k.startswith('OS_')): + self._environ_vars[k] = os.environ.pop(k) + + def _replace_swift_env_vars(self): + os.environ.update(self._environ_vars) + + +class TestParsing(TestBase): + + def setUp(self): + super(TestParsing, self).setUp() + self._remove_swift_env_vars() + + def tearDown(self): + self._replace_swift_env_vars() + super(TestParsing, self).tearDown() + + def _make_fake_command(self, result): + def fake_command(parser, args, thread_manager): + result[0], result[1] = swiftclient.shell.parse_args(parser, args) + return fake_command + + def _verify_opts(self, actual_opts, expected_opts, expected_os_opts=None, + expected_os_opts_dict=None): + """ + Check parsed options are correct. + + :param expected_opts: v1 style options. + :param expected_os_opts: openstack style options. + :param expected_os_opts_dict: openstack options that should be found in + the os_options dict. + """ + expected_os_opts = expected_os_opts or {} + expected_os_opts_dict = expected_os_opts_dict or {} + # check the expected opts are set + for key, v in expected_opts.items(): + actual = actual_opts.get(key) + self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % + (v, key, actual)) + + for key, v in expected_os_opts.items(): + actual = actual_opts.get("os_" + key) + self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % + (v, key, actual)) + + # check the os_options dict values are set + self.assertIn('os_options', actual_opts) + actual_os_opts_dict = actual_opts['os_options'] + expected_os_opts_keys = ['project_name', 'region_name', + 'tenant_name', + 'user_domain_name', 'endpoint_type', + 'object_storage_url', 'project_domain_id', + 'user_id', 'user_domain_id', 'tenant_id', + 'service_type', 'project_id', 'auth_token', + 'project_domain_name'] + for key in expected_os_opts_keys: + self.assertIn(key, actual_os_opts_dict) + cli_key = key + if key == 'object_storage_url': + # exceptions to the pattern... + cli_key = 'storage_url' + if cli_key in expected_os_opts_dict: + expect = expected_os_opts_dict[cli_key] + else: + expect = None + actual = actual_os_opts_dict[key] + self.assertEqual(expect, actual, 'Expected %s for %s, got %s' + % (expect, key, actual)) + for key in actual_os_opts_dict: + self.assertIn(key, expected_os_opts_keys) + + # check that equivalent keys have equal values + equivalents = [('os_username', 'user'), + ('os_auth_url', 'auth'), + ('os_password', 'key')] + for pair in equivalents: + self.assertEqual(actual_opts.get(pair[0]), + actual_opts.get(pair[1])) + + def test_minimum_required_args_v3(self): + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + + # username with domain is sufficient in args because keystone will + # assume user is in default domain + args = _make_args("stat", opts, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, {}) + + # check its ok to have user_id instead of username + os_opts = {"password": "secret", + "auth_url": "http://example.com:5000/v3"} + os_opts_dict = {"user_id": "user_ID"} + all_os_opts = os_opts.copy() + all_os_opts.update(os_opts_dict) + + args = _make_args("stat", opts, all_os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check no user credentials required if token and url supplied + os_opts = {} + os_opts_dict = {"storage_url": "http://example.com:8080/v1", + "auth_token": "0123abcd"} + + args = _make_args("stat", opts, os_opts_dict, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + def test_sloppy_versions(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "identity-api-version": "3.0"} + + # check os_identity_api_version=3.0 is mapped to auth_version=3 + args = _make_args("stat", {}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} # NB: not '3.0' + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check os_identity_api_version=2 is mapped to auth_version=2.0 + # A somewhat contrived scenario - we need to pass in the v1 style opts + # to prevent auth version defaulting to 2.0 due to lack of v1 style + # options. That way we can actually verify that the sloppy 2 was + # interpreted and mapped to 2.0 + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v2.0", + "identity-api-version": "2"} + opts = {"key": "secret", + "user": "user", + "auth": "http://example.com:5000/v2.0"} + args = _make_args("stat", opts, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} # NB: not '2' + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v2.0"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + def test_os_identity_api_version(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "identity-api-version": "3"} + + # check os_identity_api_version is sufficient in place of auth_version + args = _make_args("stat", {}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check again using environment variables + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that last of auth-version, os-identity-api-version is preferred + args = _make_args("stat", {}, os_opts, '-') + ['--auth-version', '2.0'] + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # now put auth_version ahead of os-identity-api-version + args = _make_args("stat", {"auth_version": "2.0"}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that OS_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'OS_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_AUTH_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0', 'OS_AUTH_VERSION': '3'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + def test_args_v3(self): + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + os_opts_dict = {"user_id": "user_ID", + "project_id": "project_ID", + "tenant_id": "tenant_ID", + "project_domain_id": "project_domain_ID", + "user_domain_id": "user_domain_ID", + "tenant_name": "tenant", + "project_name": "project", + "project_domain_name": "project_domain", + "user_domain_name": "user_domain", + "auth_token": "token", + "storage_url": "http://example.com:8080/v1", + "region_name": "region", + "service_type": "service", + "endpoint_type": "endpoint"} + all_os_opts = os_opts.copy() + all_os_opts.update(os_opts_dict) + + # check using hyphen separator + args = _make_args("stat", opts, all_os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check using underscore separator + args = _make_args("stat", opts, all_os_opts, '_') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check using environment variables + args = _make_args("stat", {}, {}) + env = _make_env(opts, all_os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check again using OS_AUTH_VERSION instead of ST_AUTH_VERSION + env = _make_env({}, all_os_opts) + env.update({'OS_AUTH_VERSION': '3'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + def test_command_args_v3(self): + result = [None, None] + fake_command = self._make_fake_command(result) + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("stat", opts, os_opts) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self.assertEqual(['stat'], result[1]) + with mock.patch('swiftclient.shell.st_stat', fake_command): + args = args + ["container_name"] + swiftclient.shell.main(args) + self.assertEqual(["stat", "container_name"], result[1]) + + def test_insufficient_args_v3(self): + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn( + 'Auth version 3 requires either OS_USERNAME or OS_USER_ID', + str(cm.exception)) + + os_opts = {"username": "user", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_PASSWORD', str(cm.exception)) + + os_opts = {"username": "user", + "password": "secret"} + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + + def test_password_prompt(self): + def do_test(opts, os_opts, auth_version): + args = _make_args("stat", opts, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + return_value='input_pwd') as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertEqual('input_pwd', result[0]['key']) + self.assertEqual('input_pwd', result[0]['os_password']) + + # ctrl-D + with self.assertRaises(SystemExit) as cm: + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + side_effect=EOFError) as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertIn( + 'Auth version %s requires' % auth_version, str(cm.exception)) + + # force getpass to think it needs to use raw input + with self.assertRaises(SystemExit) as cm: + with mock.patch('getpass.getpass', getpass.fallback_getpass): + swiftclient.shell.main(args) + self.assertIn( + 'Input stream incompatible', str(cm.exception)) + + opts = {"prompt": None, "user": "bob", "key": "secret", + "auth": "http://example.com:8080/auth/v1.0"} + do_test(opts, {}, '1.0') + os_opts = {"username": "user", + "password": "secret", + "auth_url": "http://example.com:5000/v3"} + opts = {"auth_version": "2.0", "prompt": None} + do_test(opts, os_opts, '2.0') + opts = {"auth_version": "3", "prompt": None} + do_test(opts, os_opts, '3') + + def test_no_tenant_name_or_id_v2(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "tenant_name": "", + "tenant_id": ""} + + with CaptureOutput() as output: + args = _make_args("stat", {}, os_opts) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), 'No tenant specified') + + with CaptureOutput() as output: + args = _make_args("stat", {}, os_opts, cmd_args=["testcontainer"]) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), 'No tenant specified') + + def test_no_tenant_name_or_id_v3(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "tenant_name": "", + "tenant_id": ""} + + with CaptureOutput() as output: + args = _make_args("stat", {"auth_version": "3"}, os_opts) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), + 'No project name or project id specified.') + + with CaptureOutput() as output: + args = _make_args("stat", {"auth_version": "3"}, + os_opts, cmd_args=["testcontainer"]) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), + 'No project name or project id specified.') + + def test_insufficient_env_vars_v3(self): + args = _make_args("stat", {}, {}) + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "auth_url": "http://example.com:5000/v3"} + env = _make_env(opts, os_opts) + with mock.patch.dict(os.environ, env): + self.assertRaises(SystemExit, swiftclient.shell.main, args) + + os_opts = {"username": "user", + "auth_url": "http://example.com:5000/v3"} + env = _make_env(opts, os_opts) + with mock.patch.dict(os.environ, env): + self.assertRaises(SystemExit, swiftclient.shell.main, args) + + os_opts = {"username": "user", + "password": "secret"} + env = _make_env(opts, os_opts) + with mock.patch.dict(os.environ, env): + self.assertRaises(SystemExit, swiftclient.shell.main, args) + + def test_help(self): + # --help returns condensed help message + opts = {"help": None} + os_opts = {} + args = _make_args(None, opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key <api_key>]') > 0) + self.assertEqual(-1, out.find('--os-username=<auth-user-name>')) + + # --help returns condensed help message, overrides --os-help + opts = {"help": None} + os_opts = {"help": None} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key <api_key>]') > 0) + self.assertEqual(-1, out.find('--os-username=<auth-user-name>')) + + # --os-password, --os-username and --os-auth_url should be ignored + # because --help overrides it + opts = {"help": None} + os_opts = {"help": None, + "password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key <api_key>]') > 0) + self.assertEqual(-1, out.find('--os-username=<auth-user-name>')) + + # --os-help return os options help + opts = {} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key <api_key>]') > 0) + self.assertTrue(out.find('--os-username=<auth-user-name>') > 0) + + +class TestKeystoneOptions(MockHttpTest): + """ + Tests to check that options are passed from the command line or + environment variables through to the keystone client interface. + """ + all_os_opts = {'password': 'secret', + 'username': 'user', + 'auth-url': 'http://example.com:5000/v3', + 'user-domain-name': 'userdomain', + 'user-id': 'userid', + 'user-domain-id': 'userdomainid', + 'tenant-name': 'tenantname', + 'tenant-id': 'tenantid', + 'project-name': 'projectname', + 'project-id': 'projectid', + 'project-domain-id': 'projectdomainid', + 'project-domain-name': 'projectdomain', + 'cacert': 'foo', + 'cert': 'minnie', + 'key': 'mickey'} + catalog_opts = {'service-type': 'my-object-store', + 'endpoint-type': 'public', + 'region-name': 'my-region'} + flags = ['insecure', 'debug'] + + # options that are given default values in code if missing from CLI + defaults = {'auth-version': '2.0', + 'service-type': 'object-store', + 'endpoint-type': 'publicURL'} + + def _build_os_opts(self, keys): + os_opts = {} + for k in keys: + os_opts[k] = self.all_os_opts.get(k, self.catalog_opts.get(k)) + return os_opts + + def _test_options_passed_to_keystone(self, cmd, opts, os_opts, + flags=None, use_env=False, + cmd_args=None, no_auth=False): + flags = flags or [] + if use_env: + # set up fake environment variables and make a minimal command line + env = _make_env(opts, os_opts) + args = _make_args(cmd, {}, {}, separator='-', flags=flags, + cmd_args=cmd_args) + else: + # set up empty environment and make full command line + env = {} + args = _make_args(cmd, opts, os_opts, separator='-', flags=flags, + cmd_args=cmd_args) + ks_endpoint = 'http://example.com:8080/v1/AUTH_acc' + ks_token = 'fake_auth_token' + # check correct auth version gets used + key = 'auth-version' + fake_ks = FakeKeystone(endpoint=ks_endpoint, token=ks_token) + if no_auth: + fake_ks2 = fake_ks3 = None + elif opts.get(key, self.defaults.get(key)) == '2.0': + fake_ks2 = fake_ks + fake_ks3 = None + else: + fake_ks2 = None + fake_ks3 = fake_ks + # fake_conn will check that storage_url and auth_token are as expected + endpoint = os_opts.get('storage-url', ks_endpoint) + token = os_opts.get('auth-token', ks_token) + fake_conn = self.fake_http_connection(204, headers={}, + storage_url=endpoint, + auth_token=token) + + with mock.patch('swiftclient.client.ksclient_v2', fake_ks2), \ + mock.patch('swiftclient.client.ksclient_v3', fake_ks3), \ + mock.patch('swiftclient.client.http_connection', fake_conn), \ + mock.patch.dict(os.environ, env, clear=True), \ + patch_disable_warnings() as mock_disable_warnings: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + except SwiftError as err: + self.fail('Unexpected SwiftError: %s' % err) + + if InsecureRequestWarning is not None: + if 'insecure' in flags: + self.assertEqual([mock.call(InsecureRequestWarning)], + mock_disable_warnings.mock_calls) + else: + self.assertEqual([], mock_disable_warnings.mock_calls) + + if no_auth: + # We patched out both keystoneclient versions to be None; + # they *can't* have been used and if we tried to, we would + # have raised ClientExceptions + return + + # check args passed to keystone Client __init__ + self.assertEqual(len(fake_ks.calls), 1) + actual_args = fake_ks.calls[0] + for key in self.all_os_opts.keys(): + expected = os_opts.get(key, self.defaults.get(key)) + key = key.replace('-', '_') + self.assertIn(key, actual_args) + self.assertEqual(expected, actual_args[key], + 'Expected %s for key %s, found %s' + % (expected, key, actual_args[key])) + for flag in flags: + self.assertIn(flag, actual_args) + self.assertTrue(actual_args[flag]) + + check_attr = True + # check args passed to ServiceCatalog.url_for() method + self.assertEqual(len(fake_ks.client.service_catalog.calls), 1) + actual_args = fake_ks.client.service_catalog.calls[0] + for key in self.catalog_opts.keys(): + expected = os_opts.get(key, self.defaults.get(key)) + key = key.replace('-', '_') + if key == 'region_name': + key = 'filter_value' + if expected is None: + check_attr = False + self.assertNotIn(key, actual_args) + self.assertNotIn('attr', actual_args) + continue + self.assertIn(key, actual_args) + self.assertEqual(expected, actual_args[key], + 'Expected %s for key %s, found %s' + % (expected, key, actual_args[key])) + if check_attr: + key, v = 'attr', 'region' + self.assertIn(key, actual_args) + self.assertEqual(v, actual_args[key], + 'Expected %s for key %s, found %s' + % (v, key, actual_args[key])) + + def _test_options(self, opts, os_opts, flags=None, no_auth=False): + # repeat test for different commands using env and command line options + for cmd in ('stat', 'post'): + self._test_options_passed_to_keystone(cmd, opts, os_opts, + flags=flags, no_auth=no_auth) + self._test_options_passed_to_keystone(cmd, opts, os_opts, + flags=flags, use_env=True, + no_auth=no_auth) + + def test_all_args_passed_to_keystone(self): + # check that all possible command line args are passed to keystone + opts = {'auth-version': '3'} + os_opts = dict(self.all_os_opts) + os_opts.update(self.catalog_opts) + self._test_options(opts, os_opts, flags=self.flags) + + opts = {'auth-version': '2.0'} + self._test_options(opts, os_opts, flags=self.flags) + + opts = {} + self.defaults['auth-version'] = '3' + self._test_options(opts, os_opts, flags=self.flags) + + for o in ('user-domain-name', 'user-domain-id', + 'project-domain-name', 'project-domain-id'): + os_opts.pop(o) + self.defaults['auth-version'] = '2.0' + self._test_options(opts, os_opts, flags=self.flags) + + def test_catalog_options_and_flags_not_required_v3(self): + # check that all possible command line args are passed to keystone + opts = {'auth-version': '3'} + os_opts = dict(self.all_os_opts) + self._test_options(opts, os_opts, flags=None) + + def test_ok_option_combinations_v3(self): + opts = {'auth-version': '3'} + keys = ('username', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'tenant-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'project-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'project-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + def test_ok_option_combinations_v2(self): + opts = {'auth-version': '2.0'} + keys = ('username', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'password', 'tenant-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + # allow auth_version to default to 2.0 + opts = {} + keys = ('username', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'password', 'tenant-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + # ...except when it should be 3 + self.defaults['auth-version'] = '3' + keys = ('username', 'user-domain-name', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'user-domain-id', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'project-domain-name', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'project-domain-id', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + def test_url_and_token_provided_on_command_line(self): + endpoint = 'http://alternate.com:8080/v1/AUTH_another' + token = 'alternate_auth_token' + os_opts = {'auth-token': token, + 'storage-url': endpoint} + opts = {'auth-version': '3'} + self._test_options(opts, os_opts, no_auth=True) + + opts = {'auth-version': '2.0'} + self._test_options(opts, os_opts, no_auth=True) + + def test_url_provided_on_command_line(self): + endpoint = 'http://alternate.com:8080/v1/AUTH_another' + os_opts = {'username': 'username', + 'password': 'password', + 'project-name': 'projectname', + 'auth-url': 'http://example.com:5000/v3', + 'storage-url': endpoint} + opts = {'auth-version': '3'} + self._test_options(opts, os_opts) + + opts = {'auth-version': '2.0'} + self._test_options(opts, os_opts) + + +@mock.patch.dict(os.environ, clean_os_environ) +class TestAuth(MockHttpTest): + + def test_pre_authed_request(self): + url = 'https://swift.storage.example.com/v1/AUTH_test' + token = 'AUTH_tk5b6b12' + + pre_auth_env = { + 'OS_STORAGE_URL': url, + 'OS_AUTH_TOKEN': token, + } + fake_conn = self.fake_http_connection(200) + with mock.patch('swiftclient.client.http_connection', new=fake_conn): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', {'x-auth-token': token}), + ]) + + # and again with re-auth + pre_auth_env.update(mocked_os_environ) + pre_auth_env['OS_AUTH_TOKEN'] = 'expired' + fake_conn = self.fake_http_connection(401, 200, 200, headers={ + 'x-auth-token': token + '_new', + 'x-storage-url': url + '_not_used', + }) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', { + 'x-auth-token': 'expired', + }), + ('GET', mocked_os_environ['ST_AUTH'], '', { + 'x-auth-user': mocked_os_environ['ST_USER'], + 'x-auth-key': mocked_os_environ['ST_KEY'], + }), + ('HEAD', url, '', { + 'x-auth-token': token + '_new', + }), + ]) + + def test_os_pre_authed_request(self): + url = 'https://swift.storage.example.com/v1/AUTH_test' + token = 'AUTH_tk5b6b12' + + pre_auth_env = { + 'OS_STORAGE_URL': url, + 'OS_AUTH_TOKEN': token, + } + fake_conn = self.fake_http_connection(200) + with mock.patch('swiftclient.client.http_connection', new=fake_conn): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', {'x-auth-token': token}), + ]) + + # and again with re-auth + os_environ = { + 'OS_AUTH_URL': 'https://keystone.example.com/v2.0/', + 'OS_TENANT_NAME': 'demo', + 'OS_USERNAME': 'demo', + 'OS_PASSWORD': 'admin', + } + os_environ.update(pre_auth_env) + os_environ['OS_AUTH_TOKEN'] = 'expired' + + fake_conn = self.fake_http_connection(401, 200) + fake_keystone = fake_get_auth_keystone(storage_url=url + '_not_used', + token=token + '_new') + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + get_auth_keystone=fake_keystone, + sleep=mock.DEFAULT): + with mock.patch.dict(os.environ, os_environ): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', { + 'x-auth-token': 'expired', + }), + ('HEAD', url, '', { + 'x-auth-token': token + '_new', + }), + ]) + + def test_auth(self): + headers = { + 'x-auth-token': 'AUTH_tk5b6b12', + 'x-storage-url': 'https://swift.storage.example.com/v1/AUTH_test', + } + mock_resp = self.fake_http_connection(200, headers=headers) + with mock.patch('swiftclient.client.http_connection', new=mock_resp): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'testing', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test + export OS_AUTH_TOKEN=AUTH_tk5b6b12 + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose(self): + with mock.patch('swiftclient.client.http_connection') as mock_conn: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'te$tin&', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export ST_AUTH=https://swift.storage.example.com/auth/v1.0 + export ST_USER=test:tester + export ST_KEY='te$tin&' + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_conn.mock_calls) + + def test_auth_v2(self): + os_options = {'tenant_name': 'demo'} + with mock.patch('swiftclient.client.get_auth_keystone', + new=fake_get_auth_keystone(os_options)): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', 'admin', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=http://url/ + export OS_AUTH_TOKEN=token + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose_v2(self): + with mock.patch('swiftclient.client.get_auth_keystone') \ + as mock_keystone: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', '$eKr3t', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_IDENTITY_API_VERSION=2.0 + export OS_AUTH_VERSION=2.0 + export OS_AUTH_URL=https://keystone.example.com/v2.0/ + export OS_PASSWORD='$eKr3t' + export OS_TENANT_NAME=demo + export OS_USERNAME=demo + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_keystone.mock_calls) + + +class TestCrossAccountObjectAccess(TestBase, MockHttpTest): + """ + Tests to verify use of --os-storage-url will actually + result in the object request being sent despite account + read/write access and container write access being denied. + """ + def setUp(self): + super(TestCrossAccountObjectAccess, self).setUp() + self._remove_swift_env_vars() + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file.file.write(b'01234567890123456789') + temp_file.file.flush() + self.obj = temp_file.name + self.url = 'http://alternate.com:8080/v1' + + # account tests will attempt to access + self.account = 'AUTH_alice' + + # keystone returns endpoint for another account + self.fake_ks = FakeKeystone( + endpoint='http://example.com:8080/v1/AUTH_bob', + token='bob_token') + + self.cont = 'c1' + self.cont_path = '/v1/%s/%s' % (self.account, self.cont) + self.obj_path = '%s%s' % (self.cont_path, self.obj) + + self.os_opts = {'username': 'bob', + 'password': 'password', + 'project-name': 'proj_bob', + 'auth-url': 'http://example.com:5000/v3', + 'storage-url': '%s/%s' % (self.url, self.account)} + self.opts = {'auth-version': '3'} + + def tearDown(self): + try: + os.remove(self.obj) + except OSError: + pass + self._replace_swift_env_vars() + super(TestCrossAccountObjectAccess, self).tearDown() + + def _make_cmd(self, cmd, cmd_args=None): + return _make_cmd(cmd, self.opts, self.os_opts, cmd_args=cmd_args) + + def _fake_cross_account_auth(self, read_ok, write_ok): + def on_request(method, path, *args, **kwargs): + """ + Modify response code to 200 if cross account permissions match. + """ + status = 403 + if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) and + read_ok and method in ('GET', 'HEAD')): + status = 200 + elif (path.startswith('/v1/%s/%s%s' + % (self.account, self.cont, self.obj)) and + write_ok and method in ('PUT', 'POST', 'DELETE')): + status = 200 + return status + return on_request + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_upload_bad_threads(self, mock_connection): + mock_connection.return_value.put_object.return_value = EMPTY_ETAG + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'upload', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'upload', + cmd_args=[self.cont, self.obj, '--leave-segments'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--segment-threads", "-1"]) + check_bad(["--segment-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--segment-threads", "1"]) + + def test_upload_with_read_write_access(self): + req_handler = self._fake_cross_account_auth(True, True) + fake_conn = self.fake_http_connection(403, 403, + on_request=req_handler) + + args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--leave-segments']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('PUT', self.cont_path), + ('PUT', self.obj_path)]) + self.assertEqual(self.obj[1:], out.strip()) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ + % self.cont + self.assertEqual(expected_err, out.err.strip()) + + def test_upload_with_write_only_access(self): + req_handler = self._fake_cross_account_auth(False, True) + fake_conn = self.fake_http_connection(403, 403, + on_request=req_handler) + args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--leave-segments']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + self.assertRequests([('PUT', self.cont_path), + ('PUT', self.obj_path)]) + self.assertEqual(self.obj[1:], out.strip()) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ + % self.cont + self.assertEqual(expected_err, out.err.strip()) + + def test_segment_upload_with_write_only_access(self): + req_handler = self._fake_cross_account_auth(False, True) + fake_conn = self.fake_http_connection(403, 403, 403, 403, + on_request=req_handler) + + args, env = self._make_cmd('upload', + cmd_args=[self.cont, self.obj, + '--leave-segments', + '--segment-size=10', + '--segment-container=%s' + % self.cont]) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + segment_time = getmtime(self.obj) + segment_path_0 = '%s/%f/20/10/00000000' % (self.obj_path, segment_time) + segment_path_1 = '%s/%f/20/10/00000001' % (self.obj_path, segment_time) + # Note that the order of segment PUTs cannot be asserted, so test for + # existence in request log individually + self.assert_request(('PUT', self.cont_path)) + self.assert_request(('PUT', segment_path_0)) + self.assert_request(('PUT', segment_path_1)) + self.assert_request(('PUT', self.obj_path)) + self.assertIn(self.obj[1:], out.out) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ + % self.cont + self.assertEqual(expected_err, out.err.strip()) + + def test_segment_upload_with_write_only_access_segments_container(self): + fake_conn = self.fake_http_connection( + 403, # PUT c1 + # HEAD c1 to get storage policy + StubResponse(200, headers={'X-Storage-Policy': 'foo'}), + 403, # PUT c1_segments + 201, # PUT c1_segments/...00 + 201, # PUT c1_segments/...01 + 201, # PUT c1/... + ) + + args, env = self._make_cmd('upload', + cmd_args=[self.cont, self.obj, + '--leave-segments', + '--segment-size=10']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + segment_time = getmtime(self.obj) + segment_path_0 = '%s_segments%s/%f/20/10/00000000' % ( + self.cont_path, self.obj, segment_time) + segment_path_1 = '%s_segments%s/%f/20/10/00000001' % ( + self.cont_path, self.obj, segment_time) + # Note that the order of segment PUTs cannot be asserted, so test for + # existence in request log individually + self.assert_request(('PUT', self.cont_path)) + self.assert_request(('PUT', self.cont_path + '_segments', '', { + 'X-Auth-Token': 'bob_token', + 'X-Storage-Policy': 'foo', + 'Content-Length': '0', + })) + self.assert_request(('PUT', segment_path_0)) + self.assert_request(('PUT', segment_path_1)) + self.assert_request(('PUT', self.obj_path)) + self.assertIn(self.obj[1:], out.out) + expected_err = ("Warning: failed to create container '%s': 403 Fake\n" + "Warning: failed to create container '%s': 403 Fake" + ) % (self.cont, self.cont + '_segments') + self.assertEqual(expected_err, out.err.strip()) + + def test_upload_with_no_access(self): + fake_conn = self.fake_http_connection(403, 403) + + args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--leave-segments']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + self.fail('Expected SystemExit') + except SystemExit: + pass + + self.assertRequests([('PUT', self.cont_path), + ('PUT', self.obj_path)]) + expected_err = 'Object PUT failed: http://1.2.3.4%s 403 Fake' \ + % self.obj_path + self.assertIn(expected_err, out.err) + self.assertEqual('', out) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_download_bad_threads(self, mock_connection): + mock_connection.return_value.get_object.return_value = [{}, ''] + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'download', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'download', + cmd_args=[self.cont, self.obj, '--no-download'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + + def test_download_with_read_write_access(self): + req_handler = self._fake_cross_account_auth(True, True) + fake_conn = self.fake_http_connection(403, on_request=req_handler, + etags=[EMPTY_ETAG]) + + args, env = self._make_cmd('download', cmd_args=[self.cont, + self.obj.lstrip('/'), + '--no-download']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('GET', self.obj_path)]) + self.assertTrue(out.out.startswith(self.obj.lstrip('/'))) + self.assertEqual('', out.err) + + def test_download_with_read_only_access(self): + req_handler = self._fake_cross_account_auth(True, False) + fake_conn = self.fake_http_connection(403, on_request=req_handler, + etags=[EMPTY_ETAG]) + + args, env = self._make_cmd('download', cmd_args=[self.cont, + self.obj.lstrip('/'), + '--no-download']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('GET', self.obj_path)]) + self.assertTrue(out.out.startswith(self.obj.lstrip('/'))) + self.assertEqual('', out.err) + + def test_download_with_no_access(self): + fake_conn = self.fake_http_connection(403) + args, env = self._make_cmd('download', cmd_args=[self.cont, + self.obj.lstrip('/'), + '--no-download']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + self.fail('Expected SystemExit') + except SystemExit: + pass + + self.assertRequests([('GET', self.obj_path)]) + path = '%s%s' % (self.cont, self.obj) + expected_err = "Error downloading object '%s'" % path + self.assertTrue(out.err.startswith(expected_err)) + self.assertEqual('', out) + + def test_list_with_read_access(self): + req_handler = self._fake_cross_account_auth(True, False) + resp_body = b'{}' + resp = StubResponse(403, resp_body, { + 'etag': hashlib.md5(resp_body).hexdigest()}) + fake_conn = self.fake_http_connection(resp, on_request=req_handler) + + args, env = self._make_cmd('download', cmd_args=[self.cont]) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('GET', '%s?format=json' % self.cont_path)]) + self.assertEqual('', out) + self.assertEqual('', out.err) + + def test_list_with_no_access(self): + fake_conn = self.fake_http_connection(403) + + args, env = self._make_cmd('download', cmd_args=[self.cont]) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + self.fail('Expected SystemExit') + except SystemExit: + pass + + self.assertRequests([('GET', '%s?format=json' % self.cont_path)]) + self.assertEqual('', out) + self.assertTrue(out.err.startswith('Container GET failed:')) + + +class TestCrossAccountObjectAccessUsingEnv(TestCrossAccountObjectAccess): + """ + Repeat super-class tests using environment variables rather than command + line to set options. + """ + + def _make_cmd(self, cmd, cmd_args=None): + return _make_cmd(cmd, self.opts, self.os_opts, cmd_args=cmd_args, + use_env=True) diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py new file mode 100644 index 0000000..2d45deb --- /dev/null +++ b/test/unit/test_swiftclient.py @@ -0,0 +1,3328 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import json +import logging +import mock +import six +import socket +import string +import unittest +import warnings +import tempfile +from hashlib import md5 +from six import binary_type +from six.moves.urllib.parse import urlparse +from requests.exceptions import RequestException + +from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, + FakeKeystone) + +from swiftclient.utils import EMPTY_ETAG +from swiftclient.exceptions import ClientException +from swiftclient import client as c +import swiftclient.utils +import swiftclient + + +class TestClientException(unittest.TestCase): + + def test_is_exception(self): + self.assertTrue(issubclass(c.ClientException, Exception)) + + def test_format(self): + exc = c.ClientException('something failed') + self.assertIn('something failed', str(exc)) + test_kwargs = ( + 'scheme', + 'host', + 'port', + 'path', + 'query', + 'status', + 'reason', + 'device', + 'response_content', + ) + for value in test_kwargs: + kwargs = { + 'http_%s' % value: value, + } + exc = c.ClientException('test', **kwargs) + self.assertIn(value, str(exc)) + + def test_attrs(self): + test_kwargs = ( + 'scheme', + 'host', + 'port', + 'path', + 'query', + 'status', + 'reason', + 'device', + 'response_content', + 'response_headers', + ) + for value in test_kwargs: + key = 'http_%s' % value + kwargs = {key: value} + exc = c.ClientException('test', **kwargs) + self.assertIs(True, hasattr(exc, key)) + self.assertEqual(getattr(exc, key), value) + + +class MockHttpResponse(object): + def __init__(self, status=0, headers=None, verify=False): + self.status = status + self.status_code = status + self.reason = "OK" + self.buffer = [] + self.requests_params = None + self.verify = verify + self.md5sum = md5() + self.headers = {'etag': '"%s"' % EMPTY_ETAG} + if headers: + self.headers.update(headers) + self.closed = False + + class Raw(object): + def __init__(self, headers): + self.headers = headers + + def read(self, **kw): + return "" + + def getheader(self, name, default): + return self.headers.get(name, default) + + self.raw = Raw(headers) + + def read(self): + return "" + + def close(self): + self.closed = True + + def getheader(self, name, default): + return self.headers.get(name, default) + + def getheaders(self): + return dict(self.headers).items() + + def fake_response(self): + return self + + def _fake_request(self, *arg, **kwarg): + self.status = 200 + self.requests_params = kwarg + if self.verify: + for chunk in kwarg['data']: + self.md5sum.update(chunk) + + # This simulate previous httplib implementation that would do a + # putrequest() and then use putheader() to send header. + for k, v in kwarg['headers'].items(): + self.buffer.append((k, v)) + return self.fake_response() + + +class TestHttpHelpers(MockHttpTest): + + def test_quote(self): + value = b'bytes\xff' + self.assertEqual('bytes%FF', c.quote(value)) + value = 'native string' + self.assertEqual('native%20string', c.quote(value)) + value = u'unicode string' + self.assertEqual('unicode%20string', c.quote(value)) + value = u'unicode:\xe9\u20ac' + self.assertEqual('unicode%3A%C3%A9%E2%82%AC', c.quote(value)) + + def test_parse_header_string(self): + value = b'bytes' + self.assertEqual(u'bytes', c.parse_header_string(value)) + value = u'unicode:\xe9\u20ac' + self.assertEqual(u'unicode:\xe9\u20ac', c.parse_header_string(value)) + value = 'native%20string' + self.assertEqual(u'native string', c.parse_header_string(value)) + + value = b'encoded%20bytes%E2%82%AC' + self.assertEqual(u'encoded bytes\u20ac', c.parse_header_string(value)) + value = 'encoded%20unicode%E2%82%AC' + self.assertEqual(u'encoded unicode\u20ac', + c.parse_header_string(value)) + + value = b'bad%20bytes%ff%E2%82%AC' + self.assertEqual(u'bad%20bytes%ff%E2%82%AC', + c.parse_header_string(value)) + value = u'bad%20unicode%ff\u20ac' + self.assertEqual(u'bad%20unicode%ff\u20ac', + c.parse_header_string(value)) + + value = b'really%20bad\xffbytes' + self.assertEqual(u'really%2520bad%FFbytes', + c.parse_header_string(value)) + + def test_http_connection(self): + url = 'http://www.test.com' + _junk, conn = c.http_connection(url) + self.assertIs(type(conn), c.HTTPConnection) + url = 'https://www.test.com' + _junk, conn = c.http_connection(url) + self.assertIs(type(conn), c.HTTPConnection) + url = 'ftp://www.test.com' + self.assertRaises(c.ClientException, c.http_connection, url) + + def test_encode_meta_headers(self): + headers = {'abc': '123', + u'x-container-meta-\u0394': 123, + u'x-account-meta-\u0394': 12.3, + u'x-object-meta-\u0394': True} + + r = swiftclient.encode_meta_headers(headers) + + self.assertEqual(len(headers), len(r)) + # ensure non meta headers are not encoded + self.assertIs(type(r.get('abc')), binary_type) + del r['abc'] + + for k, v in r.items(): + self.assertIs(type(k), binary_type) + self.assertIs(type(v), binary_type) + self.assertIn(v, (b'123', b'12.3', b'True')) + + def test_set_user_agent_default(self): + _junk, conn = c.http_connection('http://www.example.com') + req_headers = {} + + def my_request_handler(*a, **kw): + req_headers.update(kw.get('headers', {})) + conn._request = my_request_handler + + # test the default + conn.request('GET', '/') + ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') + self.assertTrue(ua.startswith('python-swiftclient-')) + + def test_set_user_agent_per_request_override(self): + _junk, conn = c.http_connection('http://www.example.com') + req_headers = {} + + def my_request_handler(*a, **kw): + req_headers.update(kw.get('headers', {})) + conn._request = my_request_handler + + # test if it's actually set + conn.request('GET', '/', headers={'User-Agent': 'Me'}) + ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') + self.assertEqual(ua, b'Me', req_headers) + + def test_set_user_agent_default_override(self): + _junk, conn = c.http_connection( + 'http://www.example.com', + default_user_agent='a-new-default') + req_headers = {} + + def my_request_handler(*a, **kw): + req_headers.update(kw.get('headers', {})) + conn._request = my_request_handler + + # test setting a default + conn._request = my_request_handler + conn.request('GET', '/') + ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') + self.assertEqual(ua, 'a-new-default') + + +class TestGetAuth(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf') + self.assertIsNone(url) + self.assertIsNone(token) + + def test_invalid_auth(self): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + auth_version="foo") + + def test_auth_v1(self): + c.http_connection = self.fake_http_connection(200, auth_v1=True) + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0") + self.assertEqual(url, 'storageURL') + self.assertEqual(token, 'someauthtoken') + + def test_auth_v1_insecure(self): + c.http_connection = self.fake_http_connection(200, 200, auth_v1=True) + url, token = c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', + auth_version='1.0', + insecure=True) + self.assertEqual(url, 'storageURL') + self.assertEqual(token, 'someauthtoken') + + with self.assertRaises(c.ClientException) as exc_context: + c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', auth_version='1.0') + # TODO: this test is really on validating the mock and not the + # the full plumbing into the requests's 'verify' option + self.assertIn('invalid_certificate', str(exc_context.exception)) + + def test_auth_v1_timeout(self): + # this test has some overlap with + # TestConnection.test_timeout_passed_down but is required to check that + # get_auth does the right thing when it is not passed a timeout arg + orig_http_connection = c.http_connection + timeouts = [] + + def fake_request_handler(*a, **kw): + if 'timeout' in kw: + timeouts.append(kw['timeout']) + else: + timeouts.append(None) + return MockHttpResponse( + status=200, + headers={ + 'x-auth-token': 'a_token', + 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) + + def fake_connection(*a, **kw): + url, conn = orig_http_connection(*a, **kw) + conn._request = fake_request_handler + return url, conn + + with mock.patch('swiftclient.client.http_connection', fake_connection): + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0", timeout=42.0) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0", timeout=None) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0") + + self.assertEqual(timeouts, [42.0, None, None]) + + def test_auth_v2_timeout(self): + # this test has some overlap with + # TestConnection.test_timeout_passed_down but is required to check that + # get_auth does the right thing when it is not passed a timeout arg + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0", timeout=42.0) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0", timeout=None) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0") + self.assertEqual(3, len(fake_ks.calls)) + timeouts = [call['timeout'] for call in fake_ks.calls] + self.assertEqual([42.0, None, None], timeouts) + + def test_auth_v2_with_tenant_name(self): + os_options = {'tenant_name': 'asdf'} + req_args = {'auth_version': '2.0'} + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_tenant_id(self): + os_options = {'tenant_id': 'asdf'} + req_args = {'auth_version': '2.0'} + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_project_name(self): + os_options = {'project_name': 'asdf'} + req_args = {'auth_version': '2.0'} + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_project_id(self): + os_options = {'project_id': 'asdf'} + req_args = {'auth_version': '2.0'} + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_no_tenant_name_or_tenant_id(self): + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone({})): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options={}, + auth_version='2.0') + + def test_auth_v2_with_tenant_name_none_and_tenant_id_none(self): + os_options = {'tenant_name': None, + 'tenant_id': None} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options=os_options, + auth_version='2.0') + + def test_auth_v2_with_tenant_user_in_user(self): + tenant_option = {'tenant_name': 'foo'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', + os_options={}, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_tenant_name_no_os_options(self): + tenant_option = {'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + tenant_name='asdf', + os_options={}, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_os_options(self): + os_options = {'service_type': 'object-store', + 'endpoint_type': 'internalURL', + 'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_tenant_user_in_user_no_os_options(self): + tenant_option = {'tenant_name': 'foo'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_os_region_name(self): + os_options = {'region_name': 'good-region', + 'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_no_endpoint(self): + os_options = {'region_name': 'unknown_region', + 'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, c.ClientException)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + + def test_auth_v2_ks_exception(self): + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone({}, c.ClientException)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options={}, + auth_version='2.0') + + def test_auth_v2_cacert(self): + os_options = {'tenant_name': 'foo'} + auth_url_secure = 'https://www.tests.com' + auth_url_insecure = 'https://www.tests.com/self-signed-certificate' + + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cacert='ca.pem', insecure=False) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) + + def test_auth_v2_insecure(self): + os_options = {'tenant_name': 'foo'} + auth_url_secure = 'https://www.tests.com' + auth_url_insecure = 'https://www.tests.com/invalid-certificate' + + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=True) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) + + def test_auth_v2_cert(self): + os_options = {'tenant_name': 'foo'} + auth_url_no_sslauth = 'https://www.tests.com' + auth_url_sslauth = 'https://www.tests.com/client-certificate' + + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_no_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie', cert_key='mickey') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie') + + def test_auth_v3_with_tenant_name(self): + # check the correct auth version is passed to get_auth_keystone + os_options = {'tenant_name': 'asdf'} + req_args = {'auth_version': '3'} + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="3") + + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_get_keystone_client_2_0(self): + # check the correct auth version is passed to get_auth_keystone + os_options = {'tenant_name': 'asdf'} + req_args = {'auth_version': '2.0'} + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_keystoneclient_2_0('http://www.test.com', + 'asdf', 'asdf', + os_options=os_options) + + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_get_auth_keystone_versionless(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://authurl', 'user', 'key', {}) + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionless_auth_version_set(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): + c.get_auth_keystone('http://auth_url', 'user', 'key', + {}, auth_version='2.0') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v2.0', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionful(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://auth_url/v3', 'user', 'key', + {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionful(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity/v3', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionless(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_keystone_url_some_junk_nonsense(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', + token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://blah.example.com/v2moo', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + # v2 looks sorta version-y, but it's not an exact match, so this is + # probably about just as bad as anything else we might guess at + self.assertEqual('http://blah.example.com/v2moo/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_with_session(self): + mock_session = mock.MagicMock() + mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' + mock_session.get_token.return_value = 'token' + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + session=mock_session) + self.assertEqual(url, 'http://storagehost/v1/acct') + self.assertTrue(token) + + +class TestGetAccount(MockHttpTest): + + def test_no_content(self): + c.http_connection = self.fake_http_connection(204) + value = c.get_account('http://www.test.com/v1/acct', 'asdf')[1] + self.assertEqual(value, []) + self.assertRequests([ + ('GET', '/v1/acct?format=json', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&marker=marker") + c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker') + self.assertRequests([ + ('GET', '/v1/acct?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_limit(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&limit=10") + c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10) + self.assertRequests([ + ('GET', '/v1/acct?format=json&limit=10', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_prefix(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&prefix=asdf/") + c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/') + self.assertRequests([ + ('GET', '/v1/acct?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_end_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&end_marker=end_marker") + c.get_account('http://www.test.com/v1/acct', 'asdf', + end_marker='end_marker') + self.assertRequests([ + ('GET', '/v1/acct?format=json&end_marker=end_marker', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_delimiter(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&delimiter=-") + c.get_account('http://www.test.com/v1/acct', 'asdf', + delimiter='-') + self.assertRequests([ + ('GET', '/v1/acct?format=json&delimiter=-', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + +class TestHeadAccount(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200, headers={ + 'x-account-meta-color': 'blue', + }) + resp_headers = c.head_account('http://www.tests.com', 'asdf') + self.assertEqual(resp_headers['x-account-meta-color'], 'blue') + self.assertRequests([ + ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) + ]) + + def test_server_error(self): + body = 'c' * 65 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_account('http://www.tests.com', 'asdf') + e = exc_context.exception + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_status, 500) + self.assertRequests([ + ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) + ]) + # TODO: this is a fairly brittle test of the __repr__ on the + # ClientException which should probably be in a targeted test + new_body = "[first 60 chars of response] " + body[0:60] + self.assertEqual(e.__str__()[-89:], new_body) + + +class TestPostAccount(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200, headers={ + 'X-Account-Meta-Color': 'blue', + }, body='foo') + headers = {'x-account-meta-shape': 'square'} + resp_headers, body = c.post_account( + 'http://www.tests.com/path/to/account', 'asdf', + headers, query_string='bar=baz', + data='some data') + self.assertEqual('blue', resp_headers.get('x-account-meta-color')) + self.assertEqual('foo', body) + self.assertRequests([ + ('POST', 'http://www.tests.com/path/to/account?bar=baz', + 'some data', {'x-auth-token': 'asdf', + 'x-account-meta-shape': 'square'}) + ]) + # Check that we didn't mutate the request ehader dict + self.assertEqual(headers, {'x-account-meta-shape': 'square'}) + + def test_server_error(self): + body = 'c' * 65 + c.http_connection = self.fake_http_connection(500, body=body) + with self.assertRaises(c.ClientException) as exc_mgr: + c.post_account('http://www.tests.com', 'asdf', {}) + self.assertEqual(exc_mgr.exception.http_response_content, body) + self.assertEqual(exc_mgr.exception.http_status, 500) + self.assertRequests([ + ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) + ]) + # TODO: this is a fairly brittle test of the __repr__ on the + # ClientException which should probably be in a targeted test + new_body = "[first 60 chars of response] " + body[0:60] + self.assertEqual(exc_mgr.exception.__str__()[-89:], new_body) + + +class TestGetContainer(MockHttpTest): + + def test_no_content(self): + c.http_connection = self.fake_http_connection(204) + value = c.get_container('http://www.test.com/v1/acct', 'token', + 'container')[1] + self.assertEqual(value, []) + self.assertRequests([ + ('GET', '/v1/acct/container?format=json', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&marker=marker") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + marker='marker') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_limit(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&limit=10") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + limit=10) + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&limit=10', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_prefix(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&prefix=asdf/") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + prefix='asdf/') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_delimiter(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&delimiter=/") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + delimiter='/') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&delimiter=/', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_end_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&end_marker=end_marker") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + end_marker='end_marker') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&end_marker=end_marker', + '', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}), + ]) + + def test_param_path(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&path=asdf") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + path='asdf') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&path=asdf', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_request_headers(self): + c.http_connection = self.fake_http_connection( + 204, query_string="format=json") + conn = c.http_connection('http://www.test.com') + headers = {'x-client-key': 'client key'} + c.get_container('url_is_irrelevant', 'TOKEN', 'container', + http_conn=conn, headers=headers) + self.assertRequests([ + ('GET', '/container?format=json', '', { + 'x-auth-token': 'TOKEN', + 'x-client-key': 'client key', + 'accept-encoding': 'gzip', + }), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection( + 200, query_string="format=json&hello=20", body=b'[]') + c.get_container('http://www.test.com', 'asdf', 'asdf', + query_string="hello=20") + self.assertRequests([ + ('GET', '/asdf?format=json&hello=20', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + +class TestHeadContainer(MockHttpTest): + + def test_head_ok(self): + fake_conn = self.fake_http_connection( + 200, headers={'x-container-meta-color': 'blue'}) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + resp = c.head_container('https://example.com/v1/AUTH_test', + 'token', 'container') + self.assertEqual(resp['x-container-meta-color'], 'blue') + self.assertRequests([ + ('HEAD', 'https://example.com/v1/AUTH_test/container', '', + {'x-auth-token': 'token'}), + ]) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_container('http://www.test.com', 'asdf', 'container') + e = exc_context.exception + self.assertRequests([ + ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), + ]) + self.assertEqual(e.http_status, 500) + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) + + +class TestPutContainer(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + value = c.put_container('http://www.test.com', 'token', 'container') + self.assertIsNone(value) + self.assertRequests([ + ('PUT', '/container', '', { + 'x-auth-token': 'token', + 'content-length': '0'}), + ]) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.put_container('http://www.test.com', 'token', 'container') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + self.assertRequests([ + ('PUT', '/container', '', { + 'x-auth-token': 'token', + 'content-length': '0'}), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.put_container('http://www.test.com', 'asdf', 'asdf', + query_string="hello=20") + for req in self.iter_request_log(): + self.assertEqual(req['method'], 'PUT') + self.assertEqual(req['parsed_path'].path, '/asdf') + self.assertEqual(req['parsed_path'].query, 'hello=20') + self.assertEqual(req['headers']['x-auth-token'], 'asdf') + + +class TestDeleteContainer(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + value = c.delete_container('http://www.test.com', 'token', 'container') + self.assertIsNone(value) + self.assertRequests([ + ('DELETE', '/container', '', { + 'x-auth-token': 'token'}), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.delete_container('http://www.test.com', 'token', 'container', + query_string="hello=20") + self.assertRequests([ + ('DELETE', 'http://www.test.com/container?hello=20', '', { + 'x-auth-token': 'token'}) + ]) + + +class TestGetObject(MockHttpTest): + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf', + query_string="hello=20") + self.assertRequests([ + ('GET', '/asdf/asdf?hello=20', '', { + 'x-auth-token': 'asdf'}), + ]) + + def test_get_object_as_string(self): + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = c.get_object('http://storage.example.com', 'TOKEN', + 'container_name', 'object_name') + self.assertEqual(resp, 'abcde') + + def test_request_headers(self): + c.http_connection = self.fake_http_connection(200) + conn = c.http_connection('http://www.test.com') + headers = {'Range': 'bytes=1-2'} + c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object', + http_conn=conn, headers=headers) + self.assertRequests([ + ('GET', '/container/object', '', { + 'x-auth-token': 'TOKEN', + 'range': 'bytes=1-2', + }), + ]) + + def test_response_headers(self): + c.http_connection = self.fake_http_connection( + 200, headers={'X-Utf-8-Header': b't%c3%a9st', + 'X-Non-Utf-8-Header': b'%ff', + 'X-Binary-Header': b'\xff'}) + conn = c.http_connection('http://www.test.com') + headers, data = c.get_object('url_is_irrelevant', 'TOKEN', + 'container', 'object', http_conn=conn) + self.assertEqual(u't\xe9st', headers.get('x-utf-8-header', '')) + self.assertEqual(u'%ff', headers.get('x-non-utf-8-header', '')) + self.assertEqual(u'%FF', headers.get('x-binary-header', '')) + + def test_chunk_size_read_method(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) + self.assertTrue(hasattr(resp, 'read')) + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(resp.read(None), 'de') + self.assertEqual(resp.read(), '') + + def test_chunk_size_iter(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) + self.assertTrue(hasattr(resp, 'next')) + self.assertEqual(next(resp), 'abc') + self.assertEqual(next(resp), 'de') + self.assertRaises(StopIteration, next, resp) + + def test_chunk_size_read_and_iter(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcdef') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertTrue(hasattr(resp, 'read')) + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(next(resp), 'de') + self.assertEqual(resp.read(), 'f') + self.assertRaises(StopIteration, next, resp) + self.assertEqual(resp.read(), '') + + def test_chunk_size_iter_chunked_no_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection( + 200, body='abcdef', headers={'Transfer-Encoding': 'chunked'}) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + # simulate a dropped connection + resp.resp.read() + self.assertRaises(StopIteration, next, resp) + + def test_chunk_size_iter_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection( + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}), + StubResponse(206, 'cdef', {'etag': 'some etag', + 'content-length': '4', + 'content-range': 'bytes 2-5/6'}), + StubResponse(206, 'ef', {'etag': 'some etag', + 'content-length': '2', + 'content-range': 'bytes 4-5/6'}), + ) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'cd') + self.assertEqual(2, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'ef') + self.assertEqual(3, conn.attempts) + self.assertRaises(StopIteration, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=4-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_chunk_size_iter_retry_no_range_support(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection(*[ + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}) + ] * 3) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'cd') + self.assertEqual(2, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'ef') + self.assertEqual(3, conn.attempts) + self.assertRaises(StopIteration, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=4-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_chunk_size_iter_retry_bad_range_response(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection( + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}), + StubResponse(206, 'abcdef', {'etag': 'some etag', + 'content-length': '6', + 'content-range': 'chunk 1-2/3'}) + ) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertRaises(c.ClientException, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_get_object_with_resp_chunk_size_zero(self): + def get_connection(self): + def get_auth(): + return 'http://auth.test.com', 'token' + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + self.assertIs(type(conn), c.Connection) + conn.get_auth = get_auth + self.assertEqual(conn.attempts, 0) + return conn + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = get_connection(self) + conn.get_object('container1', 'obj1', resp_chunk_size=0) + self.assertEqual(conn.attempts, 1) + + +class TestHeadObject(MockHttpTest): + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_request_headers(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + headers = {'x-client-key': 'client key'} + c.head_object('url_is_irrelevant', 'TOKEN', 'container', + 'asdf', http_conn=conn, headers=headers) + self.assertRequests([ + ('HEAD', '/container/asdf', '', { + 'x-auth-token': 'TOKEN', + 'x-client-key': 'client key', + }), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + query_string = 'foo=bar' + c.head_object('url_is_irrelevant', 'token', 'container', 'key', + http_conn=conn, query_string=query_string) + self.assertRequests([ + ('HEAD', '/container/key?foo=bar', '', {'x-auth-token': 'token'}) + ]) + + +class TestPutObject(MockHttpTest): + + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) + value = c.put_object(*args) + self.assertIsInstance(value, six.string_types) + self.assertEqual(value, EMPTY_ETAG) + self.assertRequests([ + ('PUT', '/container/obj', 'body', { + 'x-auth-token': 'TOKEN', + 'content-length': '4', + 'content-type': '' + }), + ]) + + def test_unicode_ok(self): + conn = c.http_connection(u'http://www.test.com/') + mock_file = six.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + mock_file) + text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + headers = {'X-Header1': text, + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} + + resp = MockHttpResponse() + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + value = c.put_object(*args, headers=headers, http_conn=conn) + self.assertIsInstance(value, six.string_types) + # Test for RFC-2616 encoded symbols + self.assertIn(("a-b", b".x:yz mn:fg:lp"), + resp.buffer) + # Test unicode header + self.assertIn(('x-header1', text.encode('utf8')), + resp.buffer) + + def test_chunk_warning(self): + conn = c.http_connection('http://www.test.com/') + mock_file = six.StringIO('asdf') + args = ('asdf', 'asdf', 'asdf', 'asdf', mock_file) + resp = MockHttpResponse() + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + with warnings.catch_warnings(record=True) as w: + c.put_object(*args, chunk_size=20, headers={}, http_conn=conn) + self.assertEqual(len(w), 0) + + body = 'c' * 60 + c.http_connection = self.fake_http_connection(200, body=body) + args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') + with warnings.catch_warnings(record=True) as w: + c.put_object(*args, chunk_size=20) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[-1].category, UserWarning)) + + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') + with self.assertRaises(c.ClientException) as exc_context: + c.put_object(*args) + e = exc_context.exception + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) + self.assertEqual(e.http_status, 500) + self.assertRequests([ + ('PUT', '/asdf/asdf', 'asdf', { + 'x-auth-token': 'asdf', + 'content-type': ''}), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.put_object('http://www.test.com', 'asdf', 'asdf', 'asdf', + query_string="hello=20") + for req in self.iter_request_log(): + self.assertEqual(req['method'], 'PUT') + self.assertEqual(req['parsed_path'].path, '/asdf/asdf') + self.assertEqual(req['parsed_path'].query, 'hello=20') + self.assertEqual(req['headers']['x-auth-token'], 'asdf') + + def test_raw_upload(self): + # Raw upload happens when content_length is passed to put_object + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + raw_data = b'asdf' * 256 + raw_data_len = len(raw_data) + + for kwarg in ({'headers': {'Content-Length': str(raw_data_len)}}, + {'content_length': raw_data_len}): + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + + c.put_object(url='http://www.test.com', http_conn=conn, + contents=mock_file, **kwarg) + + req_data = resp.requests_params['data'] + self.assertIs(type(req_data), swiftclient.utils.LengthWrapper) + self.assertEqual(raw_data_len, len(req_data.read())) + + def test_chunk_upload(self): + # Chunked upload happens when no content_length is passed to put_object + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + raw_data = b'asdf' * 256 + chunk_size = 16 + + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + + c.put_object(url='http://www.test.com', http_conn=conn, + contents=mock_file, chunk_size=chunk_size) + req_data = resp.requests_params['data'] + self.assertTrue(hasattr(req_data, '__iter__')) + data = b'' + for chunk in req_data: + self.assertEqual(chunk_size, len(chunk)) + data += chunk + self.assertEqual(data, raw_data) + + def test_iter_upload(self): + def data(): + for chunk in ('foo', '', 'bar'): + yield chunk + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn, + contents=data()) + req_headers = resp.requests_params['headers'] + self.assertNotIn('Content-Length', req_headers) + req_data = resp.requests_params['data'] + self.assertTrue(hasattr(req_data, '__iter__')) + # If we emit an empty chunk, requests will go ahead and send it, + # causing the server to close the connection. So make sure we don't + # do that. + self.assertEqual(['foo', 'bar'], list(req_data)) + + def test_md5_mismatch(self): + conn = c.http_connection('http://www.test.com') + resp = MockHttpResponse(status=200, verify=True, + headers={'etag': '"badresponseetag"'}) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + raw_data = b'asdf' * 256 + raw_data_md5 = md5(raw_data).hexdigest() + chunk_size = 16 + + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + + contents = swiftclient.utils.ReadableToIterable(mock_file, + md5=True) + + etag = c.put_object(url='http://www.test.com', + http_conn=conn, + contents=contents, + chunk_size=chunk_size) + + self.assertNotEqual(etag, contents.get_md5sum()) + self.assertEqual(etag, 'badresponseetag') + self.assertEqual(raw_data_md5, contents.get_md5sum()) + + def test_md5_match(self): + conn = c.http_connection('http://www.test.com') + raw_data = b'asdf' * 256 + raw_data_md5 = md5(raw_data).hexdigest() + resp = MockHttpResponse(status=200, verify=True, + headers={'etag': '"' + raw_data_md5 + '"'}) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + chunk_size = 16 + + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + contents = swiftclient.utils.ReadableToIterable(mock_file, + md5=True) + + etag = c.put_object(url='http://www.test.com', + http_conn=conn, + contents=contents, + chunk_size=chunk_size) + + self.assertEqual(raw_data_md5, contents.get_md5sum()) + self.assertEqual(etag, contents.get_md5sum()) + + def test_params(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn, + etag='1234-5678', content_type='text/plain') + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['etag'], b'1234-5678') + self.assertEqual(request_header['content-type'], b'text/plain') + + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_no_content_type_old_requests(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn) + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'') + + @mock.patch('swiftclient.requests.__version__', '2.4.0') + def test_no_content_type_new_requests(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn) + request_header = resp.requests_params['headers'] + self.assertNotIn('content-type', request_header) + + def test_content_type_in_headers(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + # title-case header + hdrs = {'Content-Type': 'text/Plain'} + c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs) + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'text/Plain') + + # method param overrides headers + c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs, + content_type='image/jpeg') + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'image/jpeg') + + +class TestPostObject(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + delete_at = 2.1 # not str! we don't know what other devs will use! + args = ('http://www.test.com', 'token', 'container', 'obj', + {'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}) + c.post_object(*args) + self.assertRequests([ + ('POST', '/container/obj', '', { + 'x-auth-token': 'token', + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}), + ]) + # Check that the request header dict didn't get mutated + self.assertEqual(args[-1], { + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at, + }) + + def test_unicode_ok(self): + conn = c.http_connection(u'http://www.test.com/') + args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + headers = {'X-Header1': text, + b'X-Header2': 'value', + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', + 'X-Object-Meta-Header-not-encoded': text, + b'X-Object-Meta-Header-encoded': 'value'} + + resp = MockHttpResponse() + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + c.post_object(*args, headers=headers, http_conn=conn) + # Test for RFC-2616 encoded symbols + self.assertIn(('a-b', b".x:yz mn:kl:qr"), resp.buffer) + # Test unicode header + self.assertIn(('x-header1', text.encode('utf8')), + resp.buffer) + self.assertIn((b'x-object-meta-header-not-encoded', + text.encode('utf8')), resp.buffer) + self.assertIn((b'x-object-meta-header-encoded', b'value'), + resp.buffer) + self.assertIn((b'x-header2', b'value'), resp.buffer) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + args = ('http://www.test.com', 'token', 'container', 'obj', {}) + with self.assertRaises(c.ClientException) as exc_context: + c.post_object(*args) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + self.assertRequests([ + ('POST', 'http://www.test.com/container/obj', '', { + 'x-auth-token': 'token', + }), + ]) + + +class TestCopyObject(MockHttpTest): + + def test_server_error(self): + c.http_connection = self.fake_http_connection(500) + self.assertRaises( + c.ClientException, c.copy_object, + 'http://www.test.com/v1/AUTH', 'asdf', 'asdf', 'asdf') + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj') + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + }), + ]) + + def test_service_token(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object('http://www.test.com/v1/AUTH', None, 'container', + 'obj', destination='/container2/obj', + service_token="TOKEN") + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Service-Token': 'TOKEN', + 'Destination': '/container2/obj', + + }), + ]) + + def test_headers(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'some-hdr': 'a', 'other-hdr': 'b'}) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'some-hdr': 'a', + 'other-hdr': 'b', + }), + ]) + + def test_fresh_metadata_default(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + '/container2/obj', {'x-fresh-metadata': 'hdr-value'}) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'hdr-value', + }), + ]) + + def test_fresh_metadata_true(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'x-fresh-metadata': 'hdr-value'}, + fresh_metadata=True) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'true', + }), + ]) + + def test_fresh_metadata_false(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'x-fresh-metadata': 'hdr-value'}, + fresh_metadata=False) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'x-auth-token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'false', + }), + ]) + + def test_no_destination(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj') + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'x-auth-token': 'token', + 'Destination': '/container/obj', + }), + ]) + + +class TestDeleteObject(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + c.delete_object('http://www.test.com', 'token', 'container', 'obj') + self.assertRequests([ + ('DELETE', 'http://www.test.com/container/obj', '', { + 'x-auth-token': 'token', + }), + ]) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.delete_object('http://www.test.com', 'token', 'container', 'obj', + query_string="hello=20") + self.assertRequests([ + ('DELETE', 'http://www.test.com/container/obj?hello=20', '', { + 'x-auth-token': 'token', + }), + ]) + + +class TestGetCapabilities(MockHttpTest): + + def test_ok(self): + conn = self.fake_http_connection(200, body=b'{}') + http_conn = conn('http://www.test.com/info') + info = c.get_capabilities(http_conn) + self.assertRequests([ + ('GET', '/info', '', {'Accept-Encoding': 'gzip'}), + ]) + self.assertEqual(info, {}) + self.assertTrue(http_conn[1].resp.has_been_read) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + conn = self.fake_http_connection( + StubResponse(500, body, headers)) + http_conn = conn('http://www.test.com/info') + with self.assertRaises(c.ClientException) as exc_context: + c.get_capabilities(http_conn) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_conn_get_capabilities_with_auth(self): + auth_headers = { + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_test' + } + auth_v1_response = StubResponse(headers=auth_headers) + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(auth_v1_response, info_response) + + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key') + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', '/auth/v1.0', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'key'}), + ('GET', 'http://storage.example.com/info', '', { + 'accept-encoding': 'gzip'}), + ]) + + def test_conn_get_capabilities_with_os_auth(self): + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_test') + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + os_options = {'project_id': 'test'} + conn = c.Connection('http://keystone.example.com/v3.0', + 'user', 'key', os_options=os_options, + auth_version=3) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_url_param(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key') + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities( + 'http://other-storage.example.com/info') + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://other-storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_preauthurl_param(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + storage_url = 'http://storage.example.com/v1/AUTH_test' + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key', preauthurl=storage_url) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_os_options(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + storage_url = 'http://storage.example.com/v1/AUTH_test' + os_options = { + 'project_id': 'test', + 'object_storage_url': storage_url, + } + conn = c.Connection('http://keystone.example.com/v3.0', + 'user', 'key', os_options=os_options, + auth_version=3) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + +class TestHTTPConnection(MockHttpTest): + + def test_bad_url_scheme(self): + url = u'www.test.com' + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception + expected = u'Unsupported scheme "" in url "www.test.com"' + self.assertEqual(expected, str(exc)) + + url = u'://www.test.com' + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception + expected = u'Unsupported scheme "" in url "://www.test.com"' + self.assertEqual(expected, str(exc)) + + url = u'blah://www.test.com' + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception + expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' + self.assertEqual(expected, str(exc)) + + def test_ok_url_scheme(self): + for scheme in ('http', 'https', 'HTTP', 'HTTPS'): + url = u'%s://www.test.com' % scheme + parsed_url, conn = c.http_connection(url) + self.assertEqual(scheme.lower(), parsed_url.scheme) + self.assertEqual(u'%s://www.test.com' % scheme, conn.url) + + def test_ok_proxy(self): + conn = c.http_connection(u'http://www.test.com/', + proxy='http://localhost:8080') + self.assertEqual(conn[1].requests_args['proxies']['http'], + 'http://localhost:8080') + + def test_bad_proxy(self): + try: + c.http_connection(u'http://www.test.com/', proxy='localhost:8080') + except c.ClientException as e: + self.assertEqual(e.msg, "Proxy's missing scheme") + + def test_cacert(self): + conn = c.http_connection(u'http://www.test.com/', + cacert='/dev/urandom') + self.assertEqual(conn[1].requests_args['verify'], '/dev/urandom') + + def test_insecure(self): + conn = c.http_connection(u'http://www.test.com/', insecure=True) + self.assertEqual(conn[1].requests_args['verify'], False) + + def test_cert(self): + conn = c.http_connection(u'http://www.test.com/', cert='minnie') + self.assertEqual(conn[1].requests_args['cert'], 'minnie') + + def test_cert_key(self): + conn = c.http_connection( + u'http://www.test.com/', cert='minnie', cert_key='mickey') + self.assertEqual(conn[1].requests_args['cert'], ('minnie', 'mickey')) + + def test_response_connection_released(self): + _parsed_url, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse() + conn.resp.raw = mock.Mock() + conn.resp.raw.read.side_effect = ["Chunk", ""] + resp = conn.getresponse() + self.assertFalse(resp.closed) + self.assertEqual("Chunk", resp.read()) + self.assertFalse(resp.read()) + self.assertTrue(resp.closed) + + @unittest.skipIf(six.PY3, 'python2 specific test') + def test_response_python2_headers(self): + '''Test utf-8 headers in Python 2. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + '\xd8\xaa-unicode': '\xd8\xaa-value', + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), + ('empty-header', ''), + ('etag', '"%s"' % EMPTY_ETAG)]), + dict(resp.getheaders())) + + @unittest.skipIf(six.PY2, 'python3 specific test') + def test_response_python3_headers(self): + '''Test latin1-encoded headers in Python 3. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + b'\xd8\xaa-unicode'.decode('iso-8859-1'): + b'\xd8\xaa-value'.decode('iso-8859-1'), + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-unicode')) + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\u062a-unicode', '\u062a-value'), + ('empty-header', ''), + ('etag', ('"%s"' % EMPTY_ETAG))]), + dict(resp.getheaders())) + + +class TestConnection(MockHttpTest): + + def test_instance(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + self.assertEqual(conn.retries, 5) + + def test_instance_kwargs(self): + args = {'user': 'ausername', + 'key': 'secretpass', + 'authurl': 'http://www.test.com', + 'tenant_name': 'atenant'} + conn = c.Connection(**args) + self.assertEqual(type(conn), c.Connection) + + def test_instance_kwargs_token(self): + args = {'preauthtoken': 'atoken123', + 'preauthurl': 'http://www.test.com:8080/v1/AUTH_123456'} + conn = c.Connection(**args) + self.assertEqual(conn.url, args['preauthurl']) + self.assertEqual(conn.token, args['preauthtoken']) + + def test_instance_kwargs_os_token(self): + storage_url = 'http://storage.example.com/v1/AUTH_test' + token = 'token' + args = { + 'os_options': { + 'object_storage_url': storage_url, + 'auth_token': token, + } + } + conn = c.Connection(**args) + self.assertEqual(conn.url, storage_url) + self.assertEqual(conn.token, token) + + def test_instance_kwargs_token_precedence(self): + storage_url = 'http://storage.example.com/v1/AUTH_test' + token = 'token' + args = { + 'preauthurl': storage_url, + 'preauthtoken': token, + 'os_options': { + 'auth_token': 'less-specific-token', + 'object_storage_url': 'less-specific-storage-url', + } + } + conn = c.Connection(**args) + self.assertEqual(conn.url, storage_url) + self.assertEqual(conn.token, token) + + def test_storage_url_override(self): + static_url = 'http://overridden.storage.url' + conn = c.Connection('http://auth.url/', 'some_user', 'some_key', + os_options={ + 'object_storage_url': static_url}) + method_signatures = ( + (conn.head_account, []), + (conn.get_account, []), + (conn.head_container, ('asdf',)), + (conn.get_container, ('asdf',)), + (conn.put_container, ('asdf',)), + (conn.delete_container, ('asdf',)), + (conn.head_object, ('asdf', 'asdf')), + (conn.get_object, ('asdf', 'asdf')), + (conn.put_object, ('asdf', 'asdf', 'asdf')), + (conn.post_object, ('asdf', 'asdf', {})), + (conn.delete_object, ('asdf', 'asdf')), + ) + + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.storage.url', 'tToken') + + for method, args in method_signatures: + c.http_connection = self.fake_http_connection( + 200, body=b'[]', storage_url=static_url) + method(*args) + self.assertEqual(len(self.request_log), 1) + for request in self.iter_request_log(): + self.assertEqual(request['parsed_path'].netloc, + 'overridden.storage.url') + self.assertEqual(request['headers']['x-auth-token'], + 'tToken') + + def test_get_capabilities(self): + conn = c.Connection() + with mock.patch('swiftclient.client.get_capabilities') as get_cap: + conn.get_capabilities('http://storage2.test.com') + parsed = get_cap.call_args[0][0][0] + self.assertEqual(parsed.path, '/info') + self.assertEqual(parsed.netloc, 'storage2.test.com') + conn.get_auth = lambda: ('http://storage.test.com/v1/AUTH_test', + 'token') + conn.get_capabilities() + parsed = get_cap.call_args[0][0][0] + self.assertEqual(parsed.path, '/info') + self.assertEqual(parsed.netloc, 'storage.test.com') + + def test_retry(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + code_iter = [500] * (conn.retries + 1) + c.http_connection = self.fake_http_connection(*code_iter) + + self.assertRaises(c.ClientException, conn.head_account) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_on_ratelimit(self): + + def quick_sleep(*args): + pass + c.sleep = quick_sleep + + # test retries + conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', + retry_on_ratelimit=True) + code_iter = [200] + [498] * (conn.retries + 1) + auth_resp_headers = { + 'x-auth-token': 'asdf', + 'x-storage-url': 'http://storage/v1/test', + } + c.http_connection = self.fake_http_connection( + *code_iter, headers=auth_resp_headers) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(conn.attempts, conn.retries + 1) + + # test default no-retry + c.http_connection = self.fake_http_connection( + 200, 498, + headers=auth_resp_headers) + conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(conn.attempts, 1) + + def test_retry_with_socket_error(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = socket.error + self.assertRaises(socket.error, conn.head_account) + self.assertEqual(mock_auth.call_count, 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_with_force_auth_retry_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(exception): + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = exception + self.assertRaises(exception, conn.head_account) + self.assertEqual(mock_auth.call_count, conn.retries + 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + do_test(socket.error) + do_test(RequestException) + + def test_retry_with_force_auth_retry_client_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(http_status, count): + + def mock_http_connection(*args, **kwargs): + raise ClientException('fake', http_status=http_status) + + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = mock_http_connection + self.assertRaises(ClientException, conn.head_account) + self.assertEqual(mock_auth.call_count, count) + self.assertEqual(conn.attempts, count) + + # sanity, in case of 401, the auth will be called only twice because of + # retried_auth mechanism + do_test(401, 2) + # others will be tried until retry limits + do_test(408, 6) + do_test(500, 6) + do_test(503, 6) + + def test_resp_read_on_server_error(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) + + def get_auth(*args, **kwargs): + return 'http://www.new.com', 'new' + conn.get_auth = get_auth + self.url, self.token = conn.get_auth() + + method_signatures = ( + (conn.head_account, []), + (conn.get_account, []), + (conn.head_container, ('asdf',)), + (conn.get_container, ('asdf',)), + (conn.put_container, ('asdf',)), + (conn.delete_container, ('asdf',)), + (conn.head_object, ('asdf', 'asdf')), + (conn.get_object, ('asdf', 'asdf')), + (conn.put_object, ('asdf', 'asdf', 'asdf')), + (conn.post_object, ('asdf', 'asdf', {})), + (conn.delete_object, ('asdf', 'asdf')), + ) + + for method, args in method_signatures: + c.http_connection = self.fake_http_connection(500) + self.assertRaises(c.ClientException, method, *args) + requests = list(self.iter_request_log()) + self.assertEqual(len(requests), 1) + for req in requests: + msg = '%s did not read resp on server error' % method.__name__ + self.assertTrue(req['resp'].has_been_read, msg) + + def test_reauth(self): + c.http_connection = self.fake_http_connection(401, 200) + + def get_auth(*args, **kwargs): + # this mock, and by extension this test are not + # representative of the unit under test. The real get_auth + # method will always return the os_option dict's + # object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return 'http://www.new.com', 'new' + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + c.sleep = swap_sleep + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + ) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + self.assertEqual(conn.url, 'http://www.new.com') + self.assertEqual(conn.token, 'new') + + def test_reauth_preauth(self): + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthurl='http://storage.example.com/v1/AUTH_test', + preauthtoken='expired') + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', + }) + fake_conn = self.fake_http_connection(401, auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), + ]) + + def test_reauth_os_preauth(self): + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': 'http://storage.example.com/v1/AUTH_test', + 'auth_token': 'expired', + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + fake_keystone = fake_get_auth_keystone(os_preauth_options) + fake_conn = self.fake_http_connection(401, 200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), + ]) + + def test_session_no_invalidate(self): + mock_session = mock.MagicMock() + mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' + mock_session.get_token.return_value = 'expired' + mock_session.invalidate.return_value = False + conn = c.Connection(session=mock_session) + fake_conn = self.fake_http_connection(401) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + self.assertRaises(c.ClientException, conn.head_account) + self.assertEqual(mock_session.get_token.mock_calls, [mock.call()]) + self.assertEqual(mock_session.invalidate.mock_calls, [mock.call()]) + + def test_session_can_invalidate(self): + mock_session = mock.MagicMock() + mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' + mock_session.get_token.side_effect = ['expired', 'token'] + mock_session.invalidate.return_value = True + conn = c.Connection(session=mock_session) + fake_conn = self.fake_http_connection(401, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/acct', '', {'x-auth-token': 'expired'}), + ('HEAD', '/v1/acct', '', {'x-auth-token': 'token'}), + ]) + self.assertEqual(mock_session.get_token.mock_calls, [ + mock.call(), mock.call()]) + self.assertEqual(mock_session.invalidate.mock_calls, [mock.call()]) + + def test_preauth_token_with_no_storage_url_requires_auth(self): + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthtoken='expired') + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', + }) + fake_conn = self.fake_http_connection(auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), + ]) + + def test_os_preauth_token_with_no_storage_url_requires_auth(self): + os_preauth_options = { + 'tenant_name': 'demo', + 'auth_token': 'expired', + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + storage_url = 'http://storage.example.com/v1/AUTH_user' + fake_keystone = fake_get_auth_keystone(storage_url=storage_url) + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), + ]) + + def test_preauth_url_trumps_auth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthurl=storage_url) + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'post_token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_post_url', + }) + fake_conn = self.fake_http_connection(auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + def test_os_preauth_url_trumps_auth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': storage_url, + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_post_url', + token='post_token') + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + def test_preauth_url_trumps_os_preauth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + os_storage_url = 'http://storage.example.com/v1/AUTH_os_pre_url' + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': os_storage_url, + } + orig_os_preauth_options = dict(os_preauth_options) + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2, + preauthurl=storage_url, tenant_name='not_demo') + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_post_url', + token='post_token') + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + # check that Connection has not modified our os_options + self.assertEqual(orig_os_preauth_options, os_preauth_options) + + def test_get_auth_sets_url_and_token(self): + with mock.patch('swiftclient.client.get_auth') as mock_get_auth: + mock_get_auth.return_value = ( + "https://storage.url/v1/AUTH_storage_acct", "AUTH_token" + ) + conn = c.Connection("https://auth.url/auth/v2.0", + "user", "passkey", tenant_name="tenant") + conn.get_auth() + self.assertEqual("https://storage.url/v1/AUTH_storage_acct", conn.url) + self.assertEqual("AUTH_token", conn.token) + + def test_timeout_passed_down(self): + # We want to avoid mocking http_connection(), and most especially + # avoid passing it down in argument. However, we cannot simply + # instantiate C=Connection(), then shim C.http_conn. Doing so would + # avoid some of the code under test (where _retry() invokes + # http_connection()), and would miss get_auth() completely. + # So, with regret, we do mock http_connection(), but with a very + # light shim that swaps out _request() as originally intended. + + orig_http_connection = c.http_connection + + timeouts = [] + + def my_request_handler(*a, **kw): + if 'timeout' in kw: + timeouts.append(kw['timeout']) + else: + timeouts.append(None) + return MockHttpResponse( + status=200, + headers={ + 'x-auth-token': 'a_token', + 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) + + def shim_connection(*a, **kw): + url, conn = orig_http_connection(*a, **kw) + conn._request = my_request_handler + return url, conn + + # v1 auth + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', timeout=33.0) + with mock.patch.multiple('swiftclient.client', + http_connection=shim_connection, + sleep=mock.DEFAULT): + conn.head_account() + + # 1 call is through get_auth, 1 call is HEAD for account + self.assertEqual(timeouts, [33.0, 33.0]) + + # v2 auth + timeouts = [] + os_options = {'tenant_name': 'tenant', 'auth_token': 'meta-token'} + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', timeout=33.0, + os_options=os_options, auth_version=2.0) + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): + with mock.patch.multiple('swiftclient.client', + http_connection=shim_connection, + sleep=mock.DEFAULT): + conn.head_account() + + # check timeout is passed to keystone client + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual(33.0, fake_ks.calls[0].get('timeout')) + # check timeout passed to HEAD for account + self.assertEqual(timeouts, [33.0]) + + # check token passed to keystone client + self.assertIn('token', fake_ks.calls[0]) + self.assertEqual('meta-token', fake_ks.calls[0].get('token')) + + def test_reset_stream(self): + + class LocalContents(object): + + def __init__(self, tell_value=0): + self.data = six.BytesIO(string.ascii_letters.encode() * 10) + self.data.seek(tell_value) + self.reads = [] + self.seeks = [] + self.tells = [] + + def tell(self): + self.tells.append(self.data.tell()) + return self.tells[-1] + + def seek(self, position, mode=0): + self.seeks.append((position, mode)) + self.data.seek(position, mode) + + def read(self, size=-1): + read_data = self.data.read(size) + self.reads.append((size, read_data)) + return read_data + + class LocalConnection(object): + + def __init__(self, parsed_url=None): + self.reason = "" + if parsed_url: + self.host = parsed_url.netloc + self.port = parsed_url.netloc + + def putrequest(self, *args, **kwargs): + self.send('PUT', *args, **kwargs) + + def putheader(self, *args, **kwargs): + return + + def endheaders(self, *args, **kwargs): + return + + def send(self, *args, **kwargs): + data = kwargs.get('data') + if data is not None: + if hasattr(data, 'read'): + data.read() + else: + for datum in data: + pass + raise socket.error('oops') + + def request(self, *args, **kwargs): + return + + def getresponse(self, *args, **kwargs): + self.status = 200 + return self + + def getheader(self, *args, **kwargs): + return 'header' + + def getheaders(self): + return [('key1', 'value1'), ('key2', 'value2')] + + def read(self, *args, **kwargs): + return '' + + def close(self): + pass + + def local_http_connection(url, proxy=None, cacert=None, + insecure=False, cert=None, cert_key=None, + ssl_compression=True, timeout=None): + parsed = urlparse(url) + return parsed, LocalConnection() + + with mock.patch.object(c, 'http_connection', local_http_connection): + conn = c.Connection('http://www.example.com', 'asdf', 'asdf', + retries=1, starting_backoff=.0001) + + contents = LocalContents() + exc = None + try: + conn.put_object('c', 'o', contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [0]) + self.assertEqual(contents.seeks, [(0, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) + self.assertEqual(str(exc), 'oops') + + contents = LocalContents(tell_value=123) + exc = None + try: + conn.put_object('c', 'o', contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) + self.assertEqual(str(exc), 'oops') + + contents = LocalContents(tell_value=123) + wrapped_contents = swiftclient.utils.LengthWrapper( + contents, 6, md5=True) + exc = None + try: + conn.put_object('c', 'o', wrapped_contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + self.assertEqual(contents.reads, [(6, b'tuvwxy')] * 2) + self.assertEqual(str(exc), 'oops') + self.assertEqual(md5(b'tuvwxy').hexdigest(), + wrapped_contents.get_md5sum()) + + contents = LocalContents() + contents.tell = None + exc = None + try: + conn.put_object('c', 'o', contents) + except c.ClientException as err: + exc = err + self.assertEqual(contents.seeks, []) + self.assertEqual(str(exc), "put_object('c', 'o', ...) failure " + "and no ability to reset contents for reupload.") + + def test_get_container(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, body=b'{}')): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.get_container('c1', prefix='p', limit=5, + headers=headers) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + 'accept-encoding': 'gzip', + }), + ]) + self.assertEqual(conn.attempts, 1) + + def test_head_container(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, body=b'{}')): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.head_container('c1', headers=headers) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('HEAD', '/v1/a/c1', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + }), + ]) + self.assertEqual(conn.attempts, 1) + + def test_head_object(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + query_string = 'foo=bar' + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.head_object('c1', 'o1', + headers=headers, query_string=query_string) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('HEAD', '/v1/a/c1/o1?foo=bar', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + }), + ]) + self.assertEqual(conn.attempts, 1) + + +class TestResponseDict(MockHttpTest): + """ + Verify handling of optional response_dict argument. + """ + calls = [('post_account', {}), + ('post_container', 'c', {}), + ('put_container', 'c'), + ('delete_container', 'c'), + ('post_object', 'c', 'o', {}), + ('put_object', 'c', 'o', 'body'), + ('copy_object', 'c', 'o'), + ('delete_object', 'c', 'o')] + + def fake_get_auth(*args, **kwargs): + return 'http://url', 'token' + + def test_response_dict_with_auth_error(self): + def bad_get_auth(*args, **kwargs): + raise c.ClientException('test') + + for call in self.calls: + resp_dict = {'test': 'should be untouched'} + with mock.patch('swiftclient.client.get_auth', + bad_get_auth): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + self.assertRaises(c.ClientException, getattr(conn, call[0]), + *call[1:], response_dict=resp_dict) + + self.assertEqual({'test': 'should be untouched'}, resp_dict) + + def test_response_dict_with_request_error(self): + for call in self.calls: + resp_dict = {'test': 'should be untouched'} + with mock.patch('swiftclient.client.get_auth', + self.fake_get_auth): + exc = c.ClientException('test') + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, exc=exc)): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + self.assertRaises(c.ClientException, + getattr(conn, call[0]), + *call[1:], + response_dict=resp_dict) + + self.assertEqual('should be untouched', resp_dict.get('test')) + self.assertEqual([{}], resp_dict.get('response_dicts')) + + def test_response_dict(self): + # test response_dict is populated and + # new list of response_dicts is created + for call in self.calls: + resp_dict = {'test': 'should be untouched'} + with mock.patch('swiftclient.client.get_auth', + self.fake_get_auth): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + getattr(conn, call[0])(*call[1:], response_dict=resp_dict) + + self.assertEqual('should be untouched', + resp_dict.pop('test', None)) + self.assertEqual('Fake', resp_dict.get('reason')) + self.assertEqual(200, resp_dict.get('status')) + self.assertIn('headers', resp_dict) + self.assertEqual('yes', resp_dict['headers'].get('x-works')) + children = resp_dict.pop('response_dicts', []) + self.assertEqual(1, len(children)) + self.assertEqual(resp_dict, children[0]) + + def test_response_dict_with_existing(self): + # check response_dict is populated and new dict is appended + # to existing response_dicts list + for call in self.calls: + resp_dict = {'test': 'should be untouched', + 'response_dicts': [{'existing': 'response dict'}]} + with mock.patch('swiftclient.client.get_auth', + self.fake_get_auth): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + getattr(conn, call[0])(*call[1:], response_dict=resp_dict) + + self.assertEqual('should be untouched', + resp_dict.pop('test', None)) + self.assertEqual('Fake', resp_dict.get('reason')) + self.assertEqual(200, resp_dict.get('status')) + self.assertIn('headers', resp_dict) + self.assertEqual('yes', resp_dict['headers'].get('x-works')) + children = resp_dict.pop('response_dicts', []) + self.assertEqual(2, len(children)) + self.assertEqual({'existing': 'response dict'}, children[0]) + self.assertEqual(resp_dict, children[1]) + + +class TestLogging(MockHttpTest): + """ + Make sure all the lines in http_log are covered. + """ + + def setUp(self): + super(TestLogging, self).setUp() + self.swiftclient_logger = logging.getLogger("swiftclient") + self.log_level = self.swiftclient_logger.getEffectiveLevel() + self.swiftclient_logger.setLevel(logging.INFO) + + def tearDown(self): + self.swiftclient_logger.setLevel(self.log_level) + super(TestLogging, self).tearDown() + + def test_put_ok(self): + c.http_connection = self.fake_http_connection(200) + args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') + value = c.put_object(*args) + self.assertIsInstance(value, six.string_types) + + def test_head_error(self): + c.http_connection = self.fake_http_connection(500) + self.assertRaises(c.ClientException, c.head_object, + 'http://www.test.com', 'asdf', 'asdf', 'asdf') + + def test_get_error(self): + c.http_connection = self.fake_http_connection(404) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 404) + + def test_content_encoding_gzip_body_is_logged_decoded(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + data = {"test": u"\u2603"} + decoded_body = json.dumps(data).encode('utf-8') + gz.write(decoded_body) + gz.close() + # stub a gzip encoded body + body = buf.getvalue() + headers = {'content-encoding': 'gzip'} + # ... and make a content-encoding gzip error response + stub_response = StubResponse(500, body, headers) + with mock.patch('swiftclient.client.logger.info') as mock_log: + # ... if the client gets such a response + c.http_connection = self.fake_http_connection(stub_response) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 500) + # it will log the decoded body + self.assertEqual([ + mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf ' + '-X GET -H "X-Auth-Token: ..."'), + mock.call('RESP STATUS: %s %s', 500, 'Fake'), + mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}), + mock.call('RESP BODY: %s', decoded_body) + ], mock_log.mock_calls) + + def test_redact_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_encoded = unicode_token_value.encode('utf8') + set_cookie_value = 'X-Auth-Token=%s' % token_value + set_cookie_encoded = set_cookie_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers={ + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded, + 'Etag': b'mock_etag', + 'Set-Cookie': set_cookie_encoded + } + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value[:16] + '...', output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value[:8] + '...', output) + self.assertIn('Set-Cookie', output) + self.assertIn(set_cookie_value[:16] + '...', output) + self.assertNotIn(token_value, output) + self.assertNotIn(unicode_token_value, output) + self.assertNotIn(set_cookie_value, output) + + def test_show_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + c.logger_settings['redact_sensitive_headers'] = False + unicode_token_encoded = unicode_token_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers=[ + ('X-Auth-Token', token_encoded), + ('X-Storage-Token', unicode_token_encoded), + ('Etag', b'mock_etag') + ] + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value, output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value, output) + + @mock.patch('swiftclient.client.logger.debug') + def test_unicode_path(self, mock_log): + path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') + c.http_log(['GET', path], {}, + MockHttpResponse(status=200, headers=[]), '') + request_log_line = mock_log.mock_calls[0] + self.assertEqual('REQ: %s', request_log_line[1][0]) + self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), + request_log_line[1][1]) + + +class TestCloseConnection(MockHttpTest): + + def test_close_none(self): + c.http_connection = self.fake_http_connection() + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + self.assertIsNone(conn.http_conn) + conn.close() + self.assertIsNone(conn.http_conn) + # Can re-close + conn.close() + self.assertIsNone(conn.http_conn) + + def test_close_ok(self): + url = 'http://www.test.com' + conn = c.Connection(url, 'asdf', 'asdf') + self.assertIsNone(conn.http_conn) + conn.http_conn = c.http_connection(url) + self.assertEqual(type(conn.http_conn), tuple) + self.assertEqual(len(conn.http_conn), 2) + http_conn_obj = conn.http_conn[1] + self.assertIsInstance(http_conn_obj, c.HTTPConnection) + self.assertTrue(hasattr(http_conn_obj, 'close')) + conn.close() + + +class TestServiceToken(MockHttpTest): + + def setUp(self): + super(TestServiceToken, self).setUp() + self.os_options = { + 'object_storage_url': 'http://storage_url.com', + 'service_username': 'service_username', + 'service_project_name': 'service_project_name', + 'service_key': 'service_key'} + + def get_connection(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + os_options=self.os_options) + + self.assertIs(type(conn), c.Connection) + conn.get_auth = self.get_auth + conn.get_service_auth = self.get_service_auth + + self.assertEqual(conn.attempts, 0) + self.assertIsNone(conn.service_token) + + self.assertIs(type(conn), c.Connection) + return conn + + def get_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'token' + + def get_service_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'stoken' + + def test_service_token_reauth(self): + get_auth_call_list = [] + + def get_auth(url, user, key, **kwargs): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} + get_auth_call_list.append(args) + return_dict = {'asdf': 'new', 'service_username': 'newserv'} + storage_url = kwargs['os_options'].get('object_storage_url') + return storage_url, return_dict[user] + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 200)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + # The original 'preauth' storage URL *must* be preserved + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'new') + self.assertEqual(conn.service_token, 'newserv') + + # Check get_auth was called with expected args + auth_args = get_auth_call_list[0] + auth_kwargs = get_auth_call_list[0]['kwargs'] + self.assertEqual('asdf', auth_args['user']) + self.assertEqual('asdf', auth_args['key']) + self.assertEqual('service_key', + auth_kwargs['os_options']['service_key']) + self.assertEqual('service_username', + auth_kwargs['os_options']['service_username']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['service_project_name']) + + auth_args = get_auth_call_list[1] + auth_kwargs = get_auth_call_list[1]['kwargs'] + self.assertEqual('service_username', auth_args['user']) + self.assertEqual('service_key', auth_args['key']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['tenant_name']) + + def test_service_token_reauth_retries_0(self): + get_auth_call_list = [] + + def get_auth(url, user, key, **kwargs): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} + get_auth_call_list.append(args) + return_dict = {'asdf': 'new', 'service_username': 'newserv'} + storage_url = kwargs['os_options'].get('object_storage_url') + return storage_url, return_dict[user] + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 200)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options, + retries=0) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + # The original 'preauth' storage URL *must* be preserved + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'new') + self.assertEqual(conn.service_token, 'newserv') + + # Check get_auth was called with expected args + auth_args = get_auth_call_list[0] + auth_kwargs = get_auth_call_list[0]['kwargs'] + self.assertEqual('asdf', auth_args['user']) + self.assertEqual('asdf', auth_args['key']) + self.assertEqual('service_key', + auth_kwargs['os_options']['service_key']) + self.assertEqual('service_username', + auth_kwargs['os_options']['service_username']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['service_project_name']) + + auth_args = get_auth_call_list[1] + auth_kwargs = get_auth_call_list[1]['kwargs'] + self.assertEqual('service_username', auth_args['user']) + self.assertEqual('service_key', auth_args['key']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['tenant_name']) + + # Ensure this is not an endless loop - it fails after the second 401 + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 401, 401, 401)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options, + retries=0) + + self.assertEqual(conn.attempts, 0) + self.assertRaises(c.ClientException, conn.head_account) + self.assertEqual(conn.attempts, 2) + unused_responses = list(self.fake_connect.code_iter) + self.assertEqual(unused_responses, [401, 401]) + + def test_service_token_get_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_account(headers={}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(204)): + conn = self.get_connection() + conn.delete_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_container_full_listing(self): + # verify service token is sent with each request for a full listing + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, 200)): + with mock.patch('swiftclient.client.parse_api_response') as resp: + resp.side_effect = ([{"name": "obj1"}], []) + conn = self.get_connection() + conn.get_container('container1', full_listing=True) + self.assertEqual(2, len(self.request_log), self.request_log) + expected_urls = iter(( + 'http://storage_url.com/container1?format=json', + 'http://storage_url.com/container1?format=json&marker=obj1' + )) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual(next(expected_urls), + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_container(self): + headers = {'X-Container-Meta-Color': 'blue'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_container('container1', headers) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + # Check that we didn't mutate the request header dict + self.assertEqual(headers, {'X-Container-Meta-Color': 'blue'}) + + def test_service_token_put_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.get_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_put_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_object('container1', 'obj1', 'a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.post_object('container1', 'obj1', {}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.delete_object('container1', 'obj1', query_string='a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1?a_string', + actual['full_path']) + self.assertEqual(conn.attempts, 1) diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py new file mode 100644 index 0000000..97abc44 --- /dev/null +++ b/test/unit/test_utils.py @@ -0,0 +1,679 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import json +import unittest +import mock +import six +import tempfile +from time import gmtime, localtime, mktime, strftime, strptime +from hashlib import md5, sha1 + +from swiftclient import utils as u + + +class TestConfigTrueValue(unittest.TestCase): + + def test_TRUE_VALUES(self): + for v in u.TRUE_VALUES: + self.assertEqual(v, v.lower()) + + @mock.patch.object(u, 'TRUE_VALUES', 'hello world'.split()) + def test_config_true_value(self): + for val in 'hello world HELLO WORLD'.split(): + self.assertIs(True, u.config_true_value(val)) + self.assertIs(True, u.config_true_value(True)) + self.assertIs(False, u.config_true_value('foo')) + self.assertIs(False, u.config_true_value(False)) + + +class TestPrtBytes(unittest.TestCase): + + def test_zero_bytes(self): + bytes_ = 0 + raw = '0' + human = '0' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_byte(self): + bytes_ = 1 + raw = '1' + human = '1' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_less_than_one_k(self): + bytes_ = (2 ** 10) - 1 + raw = '1023' + human = '1023' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_k(self): + bytes_ = 2 ** 10 + raw = '1024' + human = '1.0K' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_decimal_k(self): + bytes_ = (3 * 2 ** 10) + 512 + raw = '3584' + human = '3.5K' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_bit_less_than_one_meg(self): + bytes_ = (2 ** 20) - (2 ** 10) + raw = '1047552' + human = '1023K' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_just_a_hair_less_than_one_meg(self): + bytes_ = (2 ** 20) - (2 ** 10) + 1 + raw = '1047553' + human = '1.0M' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_meg(self): + bytes_ = 2 ** 20 + raw = '1048576' + human = '1.0M' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_ten_meg(self): + bytes_ = 10 * 2 ** 20 + human = '10M' + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_bit_less_than_ten_meg(self): + bytes_ = (10 * 2 ** 20) - (100 * 2 ** 10) + human = '9.9M' + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_just_a_hair_less_than_ten_meg(self): + bytes_ = (10 * 2 ** 20) - 1 + human = '10.0M' + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_yotta(self): + bytes_ = 42 * 2 ** 80 + self.assertEqual('42Y', u.prt_bytes(bytes_, True).lstrip()) + + def test_overflow(self): + bytes_ = 2 ** 90 + self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip()) + + +class TestTempURL(unittest.TestCase): + url = '/v1/AUTH_account/c/o' + seconds = 3600 + key = 'correcthorsebatterystaple' + method = 'GET' + expected_url = url + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600') + expected_body = '\n'.join([ + method, + '1400003600', + url, + ]).encode('utf-8') + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + self.assertEqual(url, self.expected_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + ip_ranges = [ + '1.2.3.4', '1.2.3.4/24', '2001:db8::', + b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', + ] + path = '/v1/AUTH_account/c/o/' + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_ip_range=') + for ip_range in ip_ranges: + hmac_mock.reset_mock() + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, + ip_range=ip_range) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + if isinstance(ip_range, six.binary_type): + ip_range_expected_url = ( + expected_url + ip_range.decode('utf-8') + ) + expected_body = '\n'.join([ + 'ip=' + ip_range.decode('utf-8'), + self.method, + '1400003600', + path, + ]).encode('utf-8') + else: + ip_range_expected_url = expected_url + ip_range + expected_body = '\n'.join([ + 'ip=' + ip_range, + self.method, + '1400003600', + path, + ]).encode('utf-8') + + self.assertEqual(url, ip_range_expected_url) + + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(path)) + + @mock.patch('hmac.HMAC') + def test_generate_temp_url_iso8601_argument(self, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', + self.key, self.method) + self.assertEqual(url, self.expected_url) + + # Don't care about absolute arg. + url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', + self.key, self.method, absolute=True) + self.assertEqual(url, self.expected_url) + + lt = localtime() + expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) + + if not isinstance(self.expected_url, six.string_types): + expected_url = self.expected_url.replace( + b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) + else: + expected_url = self.expected_url.replace( + '1400003600', str(int(mktime(lt)))) + url = u.generate_temp_url(self.url, expires, + self.key, self.method) + self.assertEqual(url, expected_url) + + expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) + lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) + + if not isinstance(self.expected_url, six.string_types): + expected_url = self.expected_url.replace( + b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) + else: + expected_url = self.expected_url.replace( + '1400003600', str(int(mktime(lt)))) + url = u.generate_temp_url(self.url, expires, + self.key, self.method) + self.assertEqual(url, expected_url) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method, + iso8601=True) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600)) + if not isinstance(self.url, six.string_types): + self.assertTrue(url.endswith(bytes(expires, 'utf-8'))) + else: + self.assertTrue(url.endswith(expires)) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_prefix(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + prefixes = ['', 'o', 'p0/p1/'] + for p in prefixes: + hmac_mock.reset_mock() + path = '/v1/AUTH_account/c/' + p + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_prefix=' + p) + expected_body = '\n'.join([ + self.method, + '1400003600', + 'prefix:' + path, + ]).encode('utf-8') + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, prefix=True) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + self.assertEqual(url, expected_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + + self.assertIsInstance(url, type(path)) + + def test_generate_temp_url_invalid_path(self): + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(b'/v1/a/c/\xff', self.seconds, self.key, + self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be representable as UTF-8') + + @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature") + def test_generate_absolute_expiry_temp_url(self, hmac_mock): + if isinstance(self.expected_url, six.binary_type): + expected_url = self.expected_url.replace( + b'1400003600', b'2146636800') + else: + expected_url = self.expected_url.replace( + u'1400003600', u'2146636800') + url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, + absolute=True) + self.assertEqual(url, expected_url) + + def test_generate_temp_url_bad_time(self): + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, 'not_an_int', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, -1, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, 1.1, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, '-1', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, '1.1', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, '2015-05', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url( + self.url, '2015-05-01T01:00', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + def test_generate_temp_url_bad_path(self): + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1/a/c', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('v1/a/c/o', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('blah/v1/a/c/o', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1//c/o', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1/a/c/', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1/a/c', 60, self.key, self.method, + prefix=True) + self.assertEqual(exc_manager.exception.args[0], + 'path must at least contain /v1/a/c/') + + +class TestTempURLUnicodePathAndKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3' + key = u'k\u00e9y' + expected_url = (u'%s?temp_url_sig=temp_url_signature' + u'&temp_url_expires=1400003600') % url + expected_body = u'\n'.join([ + u'GET', + u'1400003600', + url, + ]).encode('utf-8') + + +class TestTempURLUnicodePathBytesKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3' + key = u'k\u00e9y'.encode('utf-8') + expected_url = (u'%s?temp_url_sig=temp_url_signature' + u'&temp_url_expires=1400003600') % url + expected_body = '\n'.join([ + u'GET', + u'1400003600', + url, + ]).encode('utf-8') + + +class TestTempURLBytesPathUnicodeKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = u'k\u00e9y' + expected_url = url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + expected_body = b'\n'.join([ + b'GET', + b'1400003600', + url, + ]) + + +class TestTempURLBytesPathAndKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = u'k\u00e9y'.encode('utf-8') + expected_url = url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + expected_body = b'\n'.join([ + b'GET', + b'1400003600', + url, + ]) + + +class TestTempURLBytesPathAndNonUtf8Key(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = b'k\xffy' + expected_url = url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + expected_body = b'\n'.join([ + b'GET', + b'1400003600', + url, + ]) + + +class TestReadableToIterable(unittest.TestCase): + + def test_iter(self): + chunk_size = 4 + write_data = tuple(x.encode() for x in ('a', 'b', 'c', 'd')) + actual_md5sum = md5() + + with tempfile.TemporaryFile() as f: + for x in write_data: + f.write(x * chunk_size) + actual_md5sum.update(x * chunk_size) + f.seek(0) + data = u.ReadableToIterable(f, chunk_size, True) + + for i, data_chunk in enumerate(data): + self.assertEqual(chunk_size, len(data_chunk)) + self.assertEqual(data_chunk, write_data[i] * chunk_size) + + self.assertEqual(actual_md5sum.hexdigest(), data.get_md5sum()) + + def test_md5_creation(self): + # Check creation with a real and noop md5 class + data = u.ReadableToIterable(None, None, md5=True) + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + self.assertIs(type(md5()), type(data.md5sum)) + + data = u.ReadableToIterable(None, None, md5=False) + self.assertEqual('', data.get_md5sum()) + self.assertIs(u.NoopMD5, type(data.md5sum)) + + def test_unicode(self): + # Check no errors are raised if unicode data is feed in. + unicode_data = u'abc' + actual_md5sum = md5(unicode_data.encode()).hexdigest() + chunk_size = 2 + + with tempfile.TemporaryFile(mode='w+') as f: + f.write(unicode_data) + f.seek(0) + data = u.ReadableToIterable(f, chunk_size, True) + + x = next(data) + self.assertEqual(2, len(x)) + self.assertEqual(unicode_data[:2], x) + + x = next(data) + self.assertEqual(1, len(x)) + self.assertEqual(unicode_data[2:], x) + + self.assertEqual(actual_md5sum, data.get_md5sum()) + + +class TestLengthWrapper(unittest.TestCase): + + def test_stringio(self): + contents = six.StringIO(u'a' * 50 + u'b' * 50) + contents.seek(22) + data = u.LengthWrapper(contents, 42, True) + s = u'a' * 28 + u'b' * 14 + read_data = u''.join(iter(data.read, '')) + + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + + read_data = u''.join(iter(data.read, '')) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + + def test_bytesio(self): + contents = six.BytesIO(b'a' * 50 + b'b' * 50) + contents.seek(22) + data = u.LengthWrapper(contents, 42, True) + s = b'a' * 28 + b'b' * 14 + read_data = b''.join(iter(data.read, '')) + + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + def test_tempfile(self): + with tempfile.NamedTemporaryFile(mode='wb') as f: + f.write(b'a' * 100) + f.flush() + contents = open(f.name, 'rb') + data = u.LengthWrapper(contents, 42, True) + s = b'a' * 42 + read_data = b''.join(iter(data.read, '')) + + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + def test_segmented_file(self): + with tempfile.NamedTemporaryFile(mode='wb') as f: + segment_length = 1024 + segments = ('a', 'b', 'c', 'd') + for c in segments: + f.write((c * segment_length).encode()) + f.flush() + for i, c in enumerate(segments): + contents = open(f.name, 'rb') + contents.seek(i * segment_length) + data = u.LengthWrapper(contents, segment_length, True) + read_data = b''.join(iter(data.read, '')) + s = (c * segment_length).encode() + + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + read_data = b''.join(iter(data.read, '')) + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + +class TestGroupers(unittest.TestCase): + def test_n_at_a_time(self): + result = list(u.n_at_a_time(range(100), 9)) + self.assertEqual([9] * 11 + [1], list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 10)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 11)) + self.assertEqual([11] * 9 + [1], list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 12)) + self.assertEqual([12] * 8 + [4], list(map(len, result))) + + def test_n_groups(self): + result = list(u.n_groups(range(100), 9)) + self.assertEqual([12] * 8 + [4], list(map(len, result))) + + result = list(u.n_groups(range(100), 10)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_groups(range(100), 11)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_groups(range(100), 12)) + self.assertEqual([9] * 11 + [1], list(map(len, result))) + + +class TestApiResponeParser(unittest.TestCase): + + def test_utf8_default(self): + result = u.parse_api_response( + {}, u'{"test": "\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + result = u.parse_api_response( + {}, u'{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + def test_bad_json(self): + self.assertRaises(ValueError, u.parse_api_response, + {}, b'{"foo": "bar}') + + def test_bad_utf8(self): + self.assertRaises(UnicodeDecodeError, u.parse_api_response, + {}, b'{"foo": "b\xffr"}') + + def test_latin_1(self): + result = u.parse_api_response( + {'content-type': 'application/json; charset=iso8859-1'}, + b'{"t\xe9st": "\xff"}') + self.assertEqual({u't\xe9st': u'\xff'}, result) + + def test_gzipped_utf8(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + result = u.parse_api_response( + {'content-encoding': 'gzip'}, + buf.getvalue()) + self.assertEqual({'test': u'\u2603'}, result) + + +class TestGetBody(unittest.TestCase): + + def test_not_gzipped(self): + result = u.parse_api_response( + {}, u'{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + def test_gzipped_body(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + result = u.parse_api_response( + {'content-encoding': 'gzip'}, + buf.getvalue()) + self.assertEqual({'test': u'\u2603'}, result) + + +class JSONTracker(object): + def __init__(self, data): + self.data = data + self.calls = [] + + def __iter__(self): + for item in self.data: + self.calls.append(('read', item)) + yield item + + def write(self, s): + self.calls.append(('write', s)) + + +class TestJSONableIterable(unittest.TestCase): + def test_json_dump_iterencodes(self): + t = JSONTracker([1, 'fish', 2, 'fish']) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('read', 1), + ('write', '[1'), + ('read', 'fish'), + ('write', ', "fish"'), + ('read', 2), + ('write', ', 2'), + ('read', 'fish'), + ('write', ', "fish"'), + ('write', ']'), + ]) + + def test_json_dump_empty_iter(self): + t = JSONTracker([]) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('write', '[]'), + ]) diff --git a/test/unit/utils.py b/test/unit/utils.py new file mode 100644 index 0000000..025a234 --- /dev/null +++ b/test/unit/utils.py @@ -0,0 +1,582 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import functools +import sys +from requests import RequestException +from requests.structures import CaseInsensitiveDict +from time import sleep +import unittest +import mock +import six +import os +from six.moves import reload_module +from six.moves.urllib.parse import urlparse, ParseResult +from swiftclient import client as c +from swiftclient import shell as s +from swiftclient.utils import EMPTY_ETAG + + +def fake_get_auth_keystone(expected_os_options=None, exc=None, + storage_url='http://url/', token='token', + **kwargs): + def fake_get_auth_keystone(auth_url, + user, + key, + actual_os_options, **actual_kwargs): + if exc: + raise exc('test') + # TODO: some way to require auth_url, user and key? + if expected_os_options: + for key, value in actual_os_options.items(): + if value and value != expected_os_options.get(key): + return "", None + if 'required_kwargs' in kwargs: + for k, v in kwargs['required_kwargs'].items(): + if v != actual_kwargs.get(k): + return "", None + + if auth_url.startswith("https") and \ + auth_url.endswith("invalid-certificate") and \ + not actual_kwargs['insecure']: + from swiftclient import client as c + raise c.ClientException("invalid-certificate") + if auth_url.startswith("https") and \ + auth_url.endswith("self-signed-certificate") and \ + not actual_kwargs['insecure'] and \ + actual_kwargs['cacert'] is None: + from swiftclient import client as c + raise c.ClientException("unverified-certificate") + if auth_url.startswith("https") and \ + auth_url.endswith("client-certificate") and \ + not (actual_kwargs['cert'] and actual_kwargs['cert_key']): + from swiftclient import client as c + raise c.ClientException("noclient-certificate") + + return storage_url, token + return fake_get_auth_keystone + + +class StubResponse(object): + """ + Placeholder structure for use with fake_http_connect's code_iter to modify + response attributes (status, body, headers) on a per-request basis. + """ + + def __init__(self, status=200, body='', headers=None): + self.status = status + self.body = body + self.headers = headers or {} + + def __repr__(self): + return '%s(%r, %r, %r)' % (self.__class__.__name__, self.status, + self.body, self.headers) + + +def fake_http_connect(*code_iter, **kwargs): + """ + Generate a callable which yields a series of stubbed responses. Because + swiftclient will reuse an HTTP connection across pipelined requests it is + not always the case that this fake is used strictly for mocking an HTTP + connection, but rather each HTTP response (i.e. each call to requests + get_response). + """ + + class FakeConn(object): + + def __init__(self, status, etag=None, body='', timestamp='1', + headers=None): + self.status_code = self.status = status + self.reason = 'Fake' + self.scheme = 'http' + self.host = '1.2.3.4' + self.port = '1234' + self.sent = 0 + self.received = 0 + self.etag = etag + self.content = self.body = body + self.timestamp = timestamp + self.headers = headers or {} + self.request = None + + def getresponse(self): + if kwargs.get('raise_exc'): + raise Exception('test') + return self + + def getheaders(self): + if self.headers: + return self.headers.items() + headers = {'content-length': str(len(self.body)), + 'content-type': 'x-application/test', + 'x-timestamp': self.timestamp, + 'last-modified': self.timestamp, + 'x-object-meta-test': 'testing', + 'etag': + self.etag or '"%s"' % EMPTY_ETAG, + 'x-works': 'yes', + 'x-account-container-count': '12345'} + if not self.timestamp: + del headers['x-timestamp'] + try: + if next(container_ts_iter) is False: + headers['x-container-timestamp'] = '1' + except StopIteration: + pass + if 'slow' in kwargs: + headers['content-length'] = '4' + if 'headers' in kwargs: + headers.update(kwargs['headers']) + if 'auth_v1' in kwargs: + headers.update( + {'x-storage-url': 'storageURL', + 'x-auth-token': 'someauthtoken'}) + return headers.items() + + def read(self, amt=None): + if 'slow' in kwargs: + if self.sent < 4: + self.sent += 1 + sleep(0.1) + return ' ' + rv = self.body[:amt] + if amt is not None: + self.body = self.body[amt:] + else: + self.body = '' + return rv + + def send(self, amt=None): + if 'slow' in kwargs: + if self.received < 4: + self.received += 1 + sleep(0.1) + + def getheader(self, name, default=None): + return dict(self.getheaders()).get(name.lower(), default) + + def close(self): + pass + + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) + etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) + x = kwargs.get('missing_container', [False] * len(code_iter)) + if not isinstance(x, (tuple, list)): + x = [x] * len(code_iter) + container_ts_iter = iter(x) + code_iter = iter(code_iter) + + def connect(*args, **ckwargs): + if 'give_content_type' in kwargs: + if len(args) >= 7 and 'Content-Type' in args[6]: + kwargs['give_content_type'](args[6]['Content-Type']) + else: + kwargs['give_content_type']('') + if 'give_connect' in kwargs: + kwargs['give_connect'](*args, **ckwargs) + status = next(code_iter) + if isinstance(status, StubResponse): + fake_conn = FakeConn(status.status, body=status.body, + headers=status.headers) + else: + etag = next(etag_iter) + timestamp = next(timestamps_iter) + fake_conn = FakeConn(status, etag, body=kwargs.get('body', ''), + timestamp=timestamp) + if fake_conn.status <= 0: + raise RequestException() + return fake_conn + + connect.code_iter = code_iter + return connect + + +class MockHttpTest(unittest.TestCase): + + def setUp(self): + super(MockHttpTest, self).setUp() + self.fake_connect = None + self.request_log = [] + + # Capture output, since the test-runner stdout/stderr monkey-patching + # won't cover the references to sys.stdout/sys.stderr in + # swiftclient.multithreading + self.capture_output = CaptureOutput() + if 'SWIFTCLIENT_DEBUG' not in os.environ: + self.capture_output.__enter__() + self.addCleanup(self.capture_output.__exit__) + + # since we're going to steal all stderr output globally; we should + # give the developer an escape hatch or risk scorn + def blowup_but_with_the_helpful(*args, **kwargs): + raise Exception( + "You tried to enter a debugger while stderr is " + "patched, you need to set SWIFTCLIENT_DEBUG=1 " + "and try again") + import pdb + pdb.set_trace = blowup_but_with_the_helpful + + def fake_http_connection(*args, **kwargs): + self.validateMockedRequestsConsumed() + self.request_log = [] + self.fake_connect = fake_http_connect(*args, **kwargs) + _orig_http_connection = c.http_connection + query_string = kwargs.get('query_string') + storage_url = kwargs.get('storage_url') + auth_token = kwargs.get('auth_token') + exc = kwargs.get('exc') + on_request = kwargs.get('on_request') + + def wrapper(url, proxy=None, cacert=None, insecure=False, + cert=None, cert_key=None, + ssl_compression=True, timeout=None): + if storage_url: + self.assertEqual(storage_url, url) + + parsed, _conn = _orig_http_connection(url, proxy=proxy) + + class RequestsWrapper(object): + def close(self): + pass + conn = RequestsWrapper() + + def request(method, path, *args, **kwargs): + try: + conn.resp = self.fake_connect() + except StopIteration: + self.fail('Unexpected %s request for %s' % ( + method, path)) + self.request_log.append((parsed, method, path, args, + kwargs, conn.resp)) + conn.host = conn.resp.host + conn.resp.request = RequestsWrapper() + conn.resp.request.url = '%s://%s%s' % ( + conn.resp.scheme, conn.resp.host, path) + conn.resp.has_been_read = False + _orig_read = conn.resp.read + + def read(*args, **kwargs): + conn.resp.has_been_read = True + return _orig_read(*args, **kwargs) + conn.resp.read = read + if on_request: + status = on_request(method, path, *args, **kwargs) + conn.resp.status = status + if auth_token: + headers = args[1] + self.assertEqual(auth_token, + headers.get('X-Auth-Token')) + if query_string: + self.assertTrue(path.endswith('?' + query_string)) + if path.endswith('invalid_cert') and not insecure: + from swiftclient import client as c + raise c.ClientException("invalid_certificate") + if exc: + raise exc + return conn.resp + + def putrequest(path, data=None, headers=None, **kwargs): + request('PUT', path, data, headers, **kwargs) + + conn.request = request + conn.putrequest = putrequest + + def getresponse(): + return conn.resp + conn.getresponse = getresponse + + return parsed, conn + return wrapper + self.fake_http_connection = fake_http_connection + + def iter_request_log(self): + for parsed, method, path, args, kwargs, resp in self.request_log: + parts = parsed._asdict() + parts['path'] = path + full_path = ParseResult(**parts).geturl() + args = list(args) + log = dict(zip(('body', 'headers'), args)) + log.update({ + 'method': method, + 'full_path': full_path, + 'parsed_path': urlparse(full_path), + 'path': path, + 'headers': CaseInsensitiveDict(log.get('headers')), + 'resp': resp, + 'status': resp.status, + }) + yield log + + orig_assertEqual = unittest.TestCase.assertEqual + + def assert_request_equal(self, expected, real_request): + method, path = expected[:2] + if urlparse(path).scheme: + match_path = real_request['full_path'] + else: + match_path = real_request['path'] + self.assertEqual((method, path), (real_request['method'], + match_path)) + if len(expected) > 2: + body = expected[2] + real_request['expected'] = body + err_msg = 'Body mismatch for %(method)s %(path)s, ' \ + 'expected %(expected)r, and got %(body)r' % real_request + self.orig_assertEqual(body, real_request['body'], err_msg) + + if len(expected) > 3: + headers = CaseInsensitiveDict(expected[3]) + for key, value in headers.items(): + real_request['key'] = key + real_request['expected_value'] = value + real_request['value'] = real_request['headers'].get(key) + err_msg = ( + 'Header mismatch on %(key)r, ' + 'expected %(expected_value)r and got %(value)r ' + 'for %(method)s %(path)s %(headers)r' % real_request) + self.orig_assertEqual(value, real_request['value'], + err_msg) + real_request['extra_headers'] = dict( + (key, value) for key, value in real_request['headers'].items() + if key not in headers) + if real_request['extra_headers']: + self.fail('Received unexpected headers for %(method)s ' + '%(path)s, got %(extra_headers)r' % real_request) + + def assertRequests(self, expected_requests): + """ + Make sure some requests were made like you expected, provide a list of + expected requests, typically in the form of [(method, path), ...] + or [(method, path, body, headers), ...] + """ + real_requests = self.iter_request_log() + for expected in expected_requests: + real_request = next(real_requests) + self.assert_request_equal(expected, real_request) + try: + real_request = next(real_requests) + except StopIteration: + pass + else: + self.fail('At least one extra request received: %r' % + real_request) + + def assert_request(self, expected_request): + """ + Make sure a request was made as expected. Provide the + expected request in the form of [(method, path), ...] + """ + real_requests = self.iter_request_log() + for real_request in real_requests: + try: + self.assert_request_equal(expected_request, real_request) + break + except AssertionError: + pass + else: + raise AssertionError( + "Expected request %s not found in actual requests %s" + % (expected_request, self.request_log) + ) + + def validateMockedRequestsConsumed(self): + if not self.fake_connect: + return + unused_responses = list(self.fake_connect.code_iter) + if unused_responses: + self.fail('Unused responses %r' % (unused_responses,)) + + def tearDown(self): + self.validateMockedRequestsConsumed() + super(MockHttpTest, self).tearDown() + # TODO: this nuke from orbit clean up seems to be encouraging + # un-hygienic mocking on the swiftclient.client module; which may lead + # to some unfortunate test order dependency bugs by way of the broken + # window theory if any other modules are similarly patched + reload_module(c) + + +class CaptureStreamPrinter(object): + """ + CaptureStreamPrinter is used for testing unicode writing for PY3. Anything + written here is encoded as utf-8 and written to the parent CaptureStream + """ + def __init__(self, captured_stream): + self._captured_stream = captured_stream + + def write(self, data): + # No encoding, just convert the raw bytes into a str for testing + # The below call also validates that we have a byte string. + self._captured_stream.write( + data if isinstance(data, six.binary_type) else data.encode('utf8')) + + +class CaptureStream(object): + + def __init__(self, stream): + self.stream = stream + self._buffer = six.BytesIO() + self._capture = CaptureStreamPrinter(self._buffer) + self.streams = [self._capture] + + @property + def buffer(self): + if six.PY3: + return self._buffer + else: + raise AttributeError( + 'Output stream has no attribute "buffer" in Python2') + + def flush(self): + pass + + def write(self, *args, **kwargs): + for stream in self.streams: + stream.write(*args, **kwargs) + + def writelines(self, *args, **kwargs): + for stream in self.streams: + stream.writelines(*args, **kwargs) + + def getvalue(self): + return self._buffer.getvalue() + + def clear(self): + self._buffer.truncate(0) + self._buffer.seek(0) + + +class CaptureOutput(object): + + def __init__(self, suppress_systemexit=False): + self._out = CaptureStream(sys.stdout) + self._err = CaptureStream(sys.stderr) + self.patchers = [] + + WrappedOutputManager = functools.partial(s.OutputManager, + print_stream=self._out, + error_stream=self._err) + + if suppress_systemexit: + self.patchers += [ + mock.patch('swiftclient.shell.OutputManager.get_error_count', + return_value=0) + ] + + self.patchers += [ + mock.patch('swiftclient.shell.OutputManager', + WrappedOutputManager), + mock.patch('sys.stdout', self._out), + mock.patch('sys.stderr', self._err), + ] + + def __enter__(self): + for patcher in self.patchers: + patcher.start() + return self + + def __exit__(self, *args, **kwargs): + for patcher in self.patchers: + patcher.stop() + + @property + def out(self): + return self._out.getvalue().decode('utf8') + + @property + def err(self): + return self._err.getvalue().decode('utf8') + + def clear(self): + self._out.clear() + self._err.clear() + + # act like the string captured by stdout + + def __str__(self): + return self.out + + def __len__(self): + return len(self.out) + + def __eq__(self, other): + return self.out == other + + def __ne__(self, other): + return not self.__eq__(other) + + def __getattr__(self, name): + return getattr(self.out, name) + + +class FakeKeystone(object): + ''' + Fake keystone client module. Returns given endpoint url and auth token. + ''' + def __init__(self, endpoint, token): + self.calls = [] + self.auth_version = None + self.endpoint = endpoint + self.token = token + + class _Client(object): + def __init__(self, endpoint, auth_token, **kwargs): + self.auth_token = auth_token + self.endpoint = endpoint + self.service_catalog = self.ServiceCatalog(endpoint) + + class ServiceCatalog(object): + def __init__(self, endpoint): + self.calls = [] + self.endpoint_url = endpoint + + def url_for(self, **kwargs): + self.calls.append(kwargs) + return self.endpoint_url + + def Client(self, **kwargs): + self.calls.append(kwargs) + self.client = self._Client( + endpoint=self.endpoint, auth_token=self.token, **kwargs) + return self.client + + class Unauthorized(Exception): + pass + + class AuthorizationFailure(Exception): + pass + + class EndpointNotFound(Exception): + pass + + +class FakeStream(object): + def __init__(self, size): + self.bytes_read = 0 + self.size = size + + def read(self, size=-1): + if self.bytes_read == self.size: + return b'' + + if size == -1 or size + self.bytes_read > self.size: + remaining = self.size - self.bytes_read + self.bytes_read = self.size + return b'A' * remaining + + self.bytes_read += size + return b'A' * size + + def __len__(self): + return self.size |