diff options
| -rw-r--r-- | swiftclient/service.py | 53 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 91 | ||||
| -rw-r--r-- | test-requirements.txt | 5 | ||||
| -rw-r--r-- | tests/functional/test_swiftclient.py | 69 | ||||
| -rw-r--r-- | tests/sample.conf | 16 | ||||
| -rw-r--r-- | tests/unit/test_service.py | 180 | ||||
| -rw-r--r-- | tests/unit/test_shell.py | 149 | ||||
| -rw-r--r-- | tests/unit/test_swiftclient.py | 14 | ||||
| -rw-r--r-- | tests/unit/utils.py | 6 | ||||
| -rw-r--r-- | tox.ini | 1 |
10 files changed, 521 insertions, 63 deletions
diff --git a/swiftclient/service.py b/swiftclient/service.py index 2760a08..7c55769 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,6 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +import os from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -162,6 +163,8 @@ _default_local_options = { 'read_acl': None, 'write_acl': None, 'out_file': None, + 'out_directory': None, + 'remove_prefix': False, 'no_download': False, 'long': False, 'totals': False, @@ -889,7 +892,9 @@ class SwiftService(object): 'no_download': False, 'header': [], 'skip_identical': False, - 'out_file': None + 'out_directory': None, + 'out_file': None, + 'remove_prefix': False, } :returns: A generator for returning the results of the download @@ -986,6 +991,12 @@ class SwiftService(object): options['skip_identical'] = (options['skip_identical'] and out_file != '-') + if options['prefix'] and options['remove_prefix']: + path = path[len(options['prefix']):].lstrip('/') + + if options['out_directory']: + path = os.path.join(options['out_directory'], path) + if options['skip_identical']: filename = out_file if out_file else path try: @@ -1599,6 +1610,10 @@ class SwiftService(object): def _upload_object_job(self, conn, container, source, obj, options, results_queue=None): + if obj.startswith('./') or obj.startswith('.\\'): + obj = obj[2:] + if obj.startswith('/'): + obj = obj[1:] res = { 'action': 'upload_object', 'container': container, @@ -1611,10 +1626,6 @@ class SwiftService(object): path = source res['path'] = path try: - if obj.startswith('./') or obj.startswith('.\\'): - obj = obj[2:] - if obj.startswith('/'): - obj = obj[1:] if path is not None: put_headers = {'x-object-meta-mtime': "%f" % getmtime(path)} else: @@ -1815,19 +1826,19 @@ class SwiftService(object): if old_manifest or old_slo_manifest_paths: drs = [] + delobjsmap = {} if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) scontainer = unquote(scontainer) sprefix = unquote(sprefix).rstrip('/') + '/' - delobjs = [] - for delobj in conn.get_container(scontainer, - prefix=sprefix)[1]: - delobjs.append(delobj['name']) - for dr in self.delete(container=scontainer, - objects=delobjs): - drs.append(dr) + delobjsmap[scontainer] = [] + for part in self.list(scontainer, {'prefix': sprefix}): + if not part["success"]: + raise part["error"] + delobjsmap[scontainer].extend( + seg['name'] for seg in part['listing']) + if old_slo_manifest_paths: - delobjsmap = {} for seg_to_delete in old_slo_manifest_paths: if seg_to_delete in new_slo_manifest_paths: continue @@ -1836,10 +1847,18 @@ class SwiftService(object): delobjs_cont = delobjsmap.get(scont, []) delobjs_cont.append(sobj) delobjsmap[scont] = delobjs_cont - for (dscont, dsobjs) in delobjsmap.items(): - for dr in self.delete(container=dscont, - objects=dsobjs): - drs.append(dr) + + del_segs = [] + for dscont, dsobjs in delobjsmap.items(): + for dsobj in dsobjs: + del_seg = self.thread_manager.segment_pool.submit( + self._delete_segment, dscont, dsobj, + results_queue=results_queue + ) + del_segs.append(del_seg) + + for del_seg in interruptable_as_completed(del_segs): + drs.append(del_seg.result()) res['segment_delete_results'] = drs # return dict for printing diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 29b3457..3cc7314 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -31,14 +31,19 @@ from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version -from swiftclient.service import SwiftService, SwiftError, SwiftUploadObject +from swiftclient.service import SwiftService, SwiftError, \ + SwiftUploadObject, get_conn from swiftclient.command_helpers import print_account_stats, \ print_container_stats, print_object_stats +try: + from shlex import quote as sh_quote +except ImportError: + from pipes import quote as sh_quote BASENAME = 'swift' -commands = ('delete', 'download', 'list', 'post', - 'stat', 'upload', 'capabilities', 'info', 'tempurl') +commands = ('delete', 'download', 'list', 'post', 'stat', 'upload', + 'capabilities', 'info', 'tempurl', 'auth') def immediate_exit(signum, frame): @@ -146,9 +151,11 @@ def st_delete(parser, args, output_manager): st_download_options = '''[--all] [--marker] [--prefix <prefix>] - [--output <out_file>] [--object-threads <threads>] + [--output <out_file>] [--output-dir <out_directory>] + [--object-threads <threads>] [--container-threads <threads>] [--no-download] - [--skip-identical] <container> <object> + [--skip-identical] [--remove-prefix] + <container> <object> ''' st_download_help = ''' @@ -167,9 +174,15 @@ Optional arguments: --marker Marker to use when starting a container or account download. --prefix <prefix> Only download items beginning with <prefix> + --remove-prefix An optional flag for --prefix <prefix>, use this + option to download items without <prefix> --output <out_file> For a single file download, stream the output to <out_file>. Specifying "-" as <out_file> will redirect to stdout. + --output-dir <out_directory> + An optional directory to which to store objects. + By default, all objects are recreated in the current + directory. --object-threads <threads> Number of threads to use for downloading objects. Default is 10. @@ -204,6 +217,14 @@ def st_download(parser, args, output_manager): 'download, stream the output to <out_file>. ' 'Specifying "-" as <out_file> will redirect to stdout.') parser.add_option( + '-D', '--output-dir', dest='out_directory', + help='An optional directory to which to store objects. ' + 'By default, all objects are recreated in the current directory.') + parser.add_option( + '-r', '--remove-prefix', action='store_true', dest='remove_prefix', + default=False, help='An optional flag for --prefix <prefix>, ' + 'use this option to download items without <prefix>.') + parser.add_option( '', '--object-threads', type=int, default=10, help='Number of threads to use for downloading objects. ' 'Default is 10.') @@ -233,6 +254,12 @@ def st_download(parser, args, output_manager): if options.out_file and len(args) != 2: exit('-o option only allowed for single file downloads') + if not options.prefix: + options.remove_prefix = False + + if options.out_directory and len(args) == 2: + exit('Please use -o option for single file downloads and renames') + if (not args and not options.yes_all) or (args and options.yes_all): output_manager.error('Usage: %s download %s\n%s', BASENAME, st_download_options, st_download_help) @@ -763,6 +790,9 @@ def st_upload(parser, args, output_manager): return options.segment_size = str((1024 ** size_mod) * multiplier) + if int(options.segment_size) <= 0: + output_manager.error("segment-size should be positive") + return _opts = vars(options) _opts['object_uu_threads'] = options.object_threads @@ -905,6 +935,46 @@ def st_capabilities(parser, args, output_manager): st_info = st_capabilities +st_auth_help = ''' +Display auth related authentication variables in shell friendly format. + + Commands to run to export storage url and auth token into + OS_STORAGE_URL and OS_AUTH_TOKEN: + + swift auth + + Commands to append to a runcom file (e.g. ~/.bashrc, /etc/profile) for + automatic authentication: + + swift auth -v -U test:tester -K testing \ + -A http://localhost:8080/auth/v1.0 + +'''.strip('\n') + + +def st_auth(parser, args, thread_manager): + (options, args) = parse_args(parser, args) + _opts = vars(options) + if options.verbose > 1: + if options.auth_version in ('1', '1.0'): + print('export ST_AUTH=%s' % sh_quote(options.auth)) + print('export ST_USER=%s' % sh_quote(options.user)) + print('export ST_KEY=%s' % sh_quote(options.key)) + else: + print('export OS_IDENTITY_API_VERSION=%s' % sh_quote( + options.auth_version)) + print('export OS_AUTH_VERSION=%s' % sh_quote(options.auth_version)) + print('export OS_AUTH_URL=%s' % sh_quote(options.auth)) + for k, v in sorted(_opts.items()): + if v and k.startswith('os_') and \ + k not in ('os_auth_url', 'os_options'): + print('export %s=%s' % (k.upper(), sh_quote(v))) + else: + conn = get_conn(_opts) + url, token = conn.get_auth() + print('export OS_STORAGE_URL=%s' % sh_quote(url)) + print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) + st_tempurl_options = '<method> <seconds> <path> <key>' @@ -913,13 +983,13 @@ st_tempurl_help = ''' Generates a temporary URL for a Swift object. Positional arguments: - [method] An HTTP method to allow for this temporary URL. + <method> An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'. - [seconds] The amount of time in seconds the temporary URL will + <seconds> The amount of time in seconds the temporary URL will be valid for. - [path] The full path to the Swift object. Example: + <path> The full path to the Swift object. Example: /v1/AUTH_account/c/o. - [key] The secret temporary URL key set on the Swift cluster. + <key> The secret temporary URL key set on the Swift cluster. To set a key, run \'swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' '''.strip('\n') @@ -1073,7 +1143,8 @@ Positional arguments: or object. upload Uploads files or directories to the given container. capabilities List cluster capabilities. - tempurl Create a temporary URL + tempurl Create a temporary URL. + auth Display auth related environment variables. Examples: %%prog download --help diff --git a/test-requirements.txt b/test-requirements.txt index 5427920..909cb04 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,9 @@ -hacking>=0.8.0,<0.9 +hacking>=0.10.0,<0.11 coverage>=3.6 discover -mock>=1.0 +mock>=1.0;python_version!='2.6' +mock==1.0.1;python_version=='2.6' oslosphinx python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 7a31fad..2be280d 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -44,6 +44,7 @@ class TestFunctional(testtools.TestCase): '/etc/swift/test.conf') config = configparser.SafeConfigParser({'auth_version': '1'}) config.read(config_file) + self.config = config if config.has_section('func_test'): auth_host = config.get('func_test', 'auth_host') auth_port = config.getint('func_test', 'auth_port') @@ -70,15 +71,20 @@ class TestFunctional(testtools.TestCase): else: self.skip_tests = True + def _get_connection(self): + """ + Subclasses may override to use different connection setup + """ + return swiftclient.Connection( + self.auth_url, self.account_username, self.password, + auth_version=self.auth_version) + def setUp(self): super(TestFunctional, self).setUp() if self.skip_tests: self.skipTest('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG') - self.conn = swiftclient.Connection( - self.auth_url, self.account_username, self.password, - auth_version=self.auth_version) - + self.conn = self._get_connection() self.conn.put_container(self.containername) self.conn.put_container(self.containername_2) self.conn.put_object( @@ -301,3 +307,58 @@ class TestFunctional(testtools.TestCase): def test_get_capabilities(self): resp = self.conn.get_capabilities() self.assertTrue(resp.get('swift')) + + +class TestUsingKeystone(TestFunctional): + """ + Repeat tests using os_options parameter to Connection. + """ + + def _get_connection(self): + account = username = password = None + if self.auth_version not in ('2', '3'): + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS') + try: + account = self.config.get('func_test', 'account') + username = self.config.get('func_test', 'username') + password = self.config.get('func_test', 'password') + except Exception: + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + os_options = {'tenant_name': account} + return swiftclient.Connection( + self.auth_url, username, password, auth_version=self.auth_version, + os_options=os_options) + + def setUp(self): + super(TestUsingKeystone, self).setUp() + + +class TestUsingKeystoneV3(TestFunctional): + """ + Repeat tests using a keystone user with domain specified. + """ + + def _get_connection(self): + account = username = password = project_domain = user_domain = None + if self.auth_version != '3': + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS') + try: + account = self.config.get('func_test', 'account4') + username = self.config.get('func_test', 'username4') + user_domain = self.config.get('func_test', 'domain4') + project_domain = self.config.get('func_test', 'domain4') + password = self.config.get('func_test', 'password4') + except Exception: + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + + os_options = {'project_name': account, + 'project_domain_name': project_domain, + 'user_domain_name': user_domain} + return swiftclient.Connection(self.auth_url, username, password, + auth_version=self.auth_version, + os_options=os_options) + + def setUp(self): + super(TestUsingKeystoneV3, self).setUp() diff --git a/tests/sample.conf b/tests/sample.conf index 3b9b03d..95c1a47 100644 --- a/tests/sample.conf +++ b/tests/sample.conf @@ -12,7 +12,21 @@ auth_prefix = /auth/ #auth_ssl = no #auth_prefix = /v2.0/ -# Primary functional test account (needs admin access to the account) +# Primary functional test account (needs admin access to the account). +# By default the tests use a swiftclient.client.Connection instance with user +# attribute set to 'account:username' based on the options 'account' and +# 'username' specified below. This can be overridden for auth systems that +# expect a different form of user attribute by setting the option +# 'account_username'. +# account_username = test_tester account = test username = tester password = testing + +# Another user is required for keystone v3 specific tests. +# Account must be in a non-default domain. +# (Suffix '4' is used to be consistent with swift functional test config). +#account4 = test4 +#username4 = tester4 +#password4 = testing4 +#domain4 = test-domain diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 74a6ce3..339aca1 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -21,12 +21,16 @@ from hashlib import md5 from mock import Mock, PropertyMock from six.moves.queue import Queue, Empty as QueueEmptyError from six import BytesIO - import swiftclient import swiftclient.utils as utils -from swiftclient.client import Connection -from swiftclient.service import SwiftService, SwiftError - +from swiftclient.client import Connection, ClientException +from swiftclient.service import SwiftService, SwiftError,\ + SwiftUploadObject +import six +if six.PY2: + import __builtin__ as builtins +else: + import builtins clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -551,6 +555,39 @@ class TestService(testtools.TestCase): self.assertEqual('Segment size should be an integer value', exc.value) + @mock.patch('swiftclient.service.stat') + @mock.patch('swiftclient.service.getmtime', return_value=1.0) + @mock.patch('swiftclient.service.getsize', return_value=4) + @mock.patch.object(builtins, 'open', return_value=six.StringIO('asdf')) + def test_upload_with_relative_path(self, *args, **kwargs): + service = SwiftService({}) + objects = [{'path': "./test", + 'strt_indx': 2}, + {'path': os.path.join(os.getcwd(), "test"), + 'strt_indx': 1}, + {'path': ".\\test", + 'strt_indx': 2}] + for obj in objects: + with mock.patch('swiftclient.service.Connection') as mock_conn: + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value =\ + 'd41d8cd98f00b204e9800998ecf8427e' + resp_iter = service.upload( + 'c', [SwiftUploadObject(obj['path'])]) + responses = [x for x in resp_iter] + for resp in responses: + self.assertTrue(resp['success']) + self.assertEqual(2, len(responses)) + create_container_resp, upload_obj_resp = responses + self.assertEqual(create_container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + obj['path'][obj['strt_indx']:]) + self.assertEqual(upload_obj_resp['path'], obj['path']) + class TestServiceUpload(testtools.TestCase): @@ -992,6 +1029,17 @@ class TestServiceUpload(testtools.TestCase): class TestServiceDownload(testtools.TestCase): + def setUp(self): + super(TestServiceDownload, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + self.opts['no_download'] = True + self.obj_content = b'c' * 10 + self.obj_etag = md5(self.obj_content).hexdigest() + self.obj_len = len(self.obj_content) + + def _readbody(self): + yield self.obj_content + def _assertDictEqual(self, a, b, m=None): # assertDictEqual is not available in py2.6 so use a shallow check # instead @@ -1008,6 +1056,103 @@ class TestServiceDownload(testtools.TestCase): self.assertIn(k, b, m) self.assertEqual(b[k], v, m) + def test_download(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = service._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['out_directory'] = 'temp_dir' + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'temp_dir/example/test') + + def test_download_with_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example/' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_remove_prefix_and_remove_slashes(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir_and_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['out_directory'] = 'new/dir' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'new/dir/test') + def test_download_object_job_skip_identical(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) @@ -1040,6 +1185,9 @@ class TestServiceDownload(testtools.TestCase): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1092,6 +1240,9 @@ class TestServiceDownload(testtools.TestCase): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1170,6 +1321,9 @@ class TestServiceDownload(testtools.TestCase): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1231,6 +1385,9 @@ class TestServiceDownload(testtools.TestCase): 'auth_end_time': mock_conn.auth_end_time, } + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True s = SwiftService() with mock.patch('swiftclient.service.time', side_effect=range(3)): with mock.patch('swiftclient.service.get_conn', @@ -1239,11 +1396,7 @@ class TestServiceDownload(testtools.TestCase): conn=mock_conn, container='test_c', obj='test_o', - options={'out_file': f.name, - 'header': {}, - 'no_download': True, - 'yes_all': False, - 'skip_identical': True}) + options=options) self._assertDictEqual(r, expected_r) @@ -1323,6 +1476,9 @@ class TestServiceDownload(testtools.TestCase): 'auth_end_time': mock_conn.auth_end_time, } + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True s = SwiftService() with mock.patch('swiftclient.service.time', side_effect=range(3)): with mock.patch('swiftclient.service.get_conn', @@ -1331,11 +1487,7 @@ class TestServiceDownload(testtools.TestCase): conn=mock_conn, container='test_c', obj='test_o', - options={'out_file': f.name, - 'header': {}, - 'no_download': True, - 'yes_all': False, - 'skip_identical': True}) + options=options) self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 98cef85..1d0e14a 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -19,8 +19,10 @@ import mock import os import tempfile import unittest +import textwrap from testtools import ExpectedException + import six import swiftclient @@ -46,6 +48,11 @@ mocked_os_environ = { 'ST_USER': 'test:tester', 'ST_KEY': 'testing' } +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -491,11 +498,11 @@ class TestShell(unittest.TestCase): expected_delete_calls = [ mock.call( b'container1', b'old_seg1', - query_string=None, response_dict={} + response_dict={} ), mock.call( b'container2', b'old_seg2', - query_string=None, response_dict={} + response_dict={} ) ] self.assertEqual( @@ -540,9 +547,11 @@ class TestShell(unittest.TestCase): ] connection.return_value.get_container.side_effect = [ [None, [{'name': 'prefix_a', 'bytes': 0, - 'last_modified': '123T456'}, - {'name': 'prefix_b', 'bytes': 0, - 'last_modified': '123T456'}]] + 'last_modified': '123T456'}]], + # Have multiple pages worth of DLO segments + [None, [{'name': 'prefix_b', 'bytes': 0, + 'last_modified': '123T456'}]], + [None, []] ] connection.return_value.put_object.return_value = ( 'd41d8cd98f00b204e9800998ecf8427e') @@ -557,11 +566,11 @@ class TestShell(unittest.TestCase): expected_delete_calls = [ mock.call( 'container1', 'prefix_a', - query_string=None, response_dict={} + response_dict={} ), mock.call( 'container1', 'prefix_b', - query_string=None, response_dict={} + response_dict={} ) ] self.assertEqual( @@ -897,6 +906,29 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) self.assertEquals(output.err, "Invalid segment size\n") + def test_negative_upload_segment_size(self): + with CaptureOutput() as output: + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40M", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40G", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + class TestSubcommandHelp(unittest.TestCase): @@ -1529,6 +1561,101 @@ class TestAuth(MockHttpTest): }), ]) + def test_auth(self): + headers = { + 'x-auth-token': 'AUTH_tk5b6b12', + 'x-storage-url': 'https://swift.storage.example.com/v1/AUTH_test', + } + mock_resp = self.fake_http_connection(200, headers=headers) + with mock.patch('swiftclient.client.http_connection', new=mock_resp): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'testing', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test + export OS_AUTH_TOKEN=AUTH_tk5b6b12 + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose(self): + with mock.patch('swiftclient.client.http_connection') as mock_conn: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'te$tin&', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export ST_AUTH=https://swift.storage.example.com/auth/v1.0 + export ST_USER=test:tester + export ST_KEY='te$tin&' + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_conn.mock_calls) + + def test_auth_v2(self): + os_options = {'tenant_name': 'demo'} + with mock.patch('swiftclient.client.get_auth_keystone', + new=fake_get_auth_keystone(os_options)): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', 'admin', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=http://url/ + export OS_AUTH_TOKEN=token + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose_v2(self): + with mock.patch('swiftclient.client.get_auth_keystone') \ + as mock_keystone: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', '$eKr3t', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_IDENTITY_API_VERSION=2.0 + export OS_AUTH_VERSION=2.0 + export OS_AUTH_URL=https://keystone.example.com/v2.0/ + export OS_PASSWORD='$eKr3t' + export OS_TENANT_NAME=demo + export OS_USERNAME=demo + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_keystone.mock_calls) + class TestCrossAccountObjectAccess(TestBase, MockHttpTest): """ @@ -1610,7 +1737,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) - self.assertEqual(self.obj, out.strip()) + self.assertEqual(self.obj[1:], out.strip()) expected_err = 'Warning: failed to create container %r: 403 Fake' \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1619,7 +1746,6 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): req_handler = self._fake_cross_account_auth(False, True) fake_conn = self.fake_http_connection(403, 403, on_request=req_handler) - args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) with mock.patch('swiftclient.client._import_keystone_client', @@ -1631,10 +1757,9 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): swiftclient.shell.main(args) except SystemExit as e: self.fail('Unexpected SystemExit: %s' % e) - self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) - self.assertEqual(self.obj, out.strip()) + self.assertEqual(self.obj[1:], out.strip()) expected_err = 'Warning: failed to create container %r: 403 Fake' \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1669,7 +1794,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): self.assert_request(('PUT', segment_path_0)) self.assert_request(('PUT', segment_path_1)) self.assert_request(('PUT', self.obj_path)) - self.assertTrue(self.obj in out.out) + self.assertTrue(self.obj[1:] in out.out) expected_err = 'Warning: failed to create container %r: 403 Fake' \ % self.cont self.assertEqual(expected_err, out.err.strip()) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index ae46099..1cfe204 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -75,6 +75,7 @@ class MockHttpResponse(object): self.headers = {'etag': '"%s"' % EMPTY_ETAG} if headers: self.headers.update(headers) + self.closed = False class Raw(object): def __init__(self, headers): @@ -92,7 +93,7 @@ class MockHttpResponse(object): return "" def close(self): - pass + self.closed = True def getheader(self, name, default): return self.headers.get(name, default) @@ -1145,6 +1146,17 @@ class TestHTTPConnection(MockHttpTest): conn = c.http_connection(u'http://www.test.com/', insecure=True) self.assertEqual(conn[1].requests_args['verify'], False) + def test_response_connection_released(self): + _parsed_url, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse() + conn.resp.raw = mock.Mock() + conn.resp.raw.read.side_effect = ["Chunk", ""] + resp = conn.getresponse() + self.assertFalse(resp.closed) + self.assertEqual("Chunk", resp.read()) + self.assertFalse(resp.read()) + self.assertTrue(resp.closed) + class TestConnection(MockHttpTest): diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 955296e..0a45437 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -38,8 +38,10 @@ def fake_get_auth_keystone(expected_os_options=None, exc=None, if exc: raise exc('test') # TODO: some way to require auth_url, user and key? - if expected_os_options and actual_os_options != expected_os_options: - return "", None + if expected_os_options: + for key, value in actual_os_options.items(): + if value and value != expected_os_options.get(key): + return "", None if 'required_kwargs' in kwargs: for k, v in kwargs['required_kwargs'].items(): if v != actual_kwargs.get(k): @@ -11,6 +11,7 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = python setup.py testr --testr-args="{posargs}" +passenv = SWIFT_* *_proxy [testenv:pep8] commands = |