diff options
| -rw-r--r-- | doc/manpages/swift.1 | 8 | ||||
| -rw-r--r-- | swiftclient/service.py | 13 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 88 | ||||
| -rw-r--r-- | test-requirements.txt | 3 | ||||
| -rw-r--r-- | tests/unit/test_service.py | 135 | ||||
| -rw-r--r-- | tests/unit/test_shell.py | 102 | ||||
| -rw-r--r-- | tests/unit/utils.py | 6 |
7 files changed, 331 insertions, 24 deletions
diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index a470432..446ade4 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -104,6 +104,14 @@ is not provided the storage-url retrieved after authentication is used as proxy-url. .RE +\fBtempurl\fR method seconds path key +.RS 4 +Generates a temporary URL allowing unauthenticated access to the Swift object at +the given path, using the given HTTP method, for the given number of seconds, +using the given TempURL key. \fBExample\fR: tempurl GET 86400 +/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key +.RE + .SH OPTIONS .PD 0 .IP "--version Show program's version number and exit" diff --git a/swiftclient/service.py b/swiftclient/service.py index d7c05ca..7c55769 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,6 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +import os from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -162,6 +163,8 @@ _default_local_options = { 'read_acl': None, 'write_acl': None, 'out_file': None, + 'out_directory': None, + 'remove_prefix': False, 'no_download': False, 'long': False, 'totals': False, @@ -889,7 +892,9 @@ class SwiftService(object): 'no_download': False, 'header': [], 'skip_identical': False, - 'out_file': None + 'out_directory': None, + 'out_file': None, + 'remove_prefix': False, } :returns: A generator for returning the results of the download @@ -986,6 +991,12 @@ class SwiftService(object): options['skip_identical'] = (options['skip_identical'] and out_file != '-') + if options['prefix'] and options['remove_prefix']: + path = path[len(options['prefix']):].lstrip('/') + + if options['out_directory']: + path = os.path.join(options['out_directory'], path) + if options['skip_identical']: filename = out_file if out_file else path try: diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 0663e4f..a77ea07 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -32,14 +32,19 @@ from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version -from swiftclient.service import SwiftService, SwiftError, SwiftUploadObject +from swiftclient.service import SwiftService, SwiftError, \ + SwiftUploadObject, get_conn from swiftclient.command_helpers import print_account_stats, \ print_container_stats, print_object_stats +try: + from shlex import quote as sh_quote +except ImportError: + from pipes import quote as sh_quote BASENAME = 'swift' -commands = ('delete', 'download', 'list', 'post', - 'stat', 'upload', 'capabilities', 'info', 'tempurl') +commands = ('delete', 'download', 'list', 'post', 'stat', 'upload', + 'capabilities', 'info', 'tempurl', 'auth') def immediate_exit(signum, frame): @@ -147,9 +152,11 @@ def st_delete(parser, args, output_manager): st_download_options = '''[--all] [--marker] [--prefix <prefix>] - [--output <out_file>] [--object-threads <threads>] + [--output <out_file>] [--output-dir <out_directory>] + [--object-threads <threads>] [--container-threads <threads>] [--no-download] - [--skip-identical] <container> <object> + [--skip-identical] [--remove-prefix] + <container> <object> ''' st_download_help = ''' @@ -168,9 +175,15 @@ Optional arguments: --marker Marker to use when starting a container or account download. --prefix <prefix> Only download items beginning with <prefix> + --remove-prefix An optional flag for --prefix <prefix>, use this + option to download items without <prefix> --output <out_file> For a single file download, stream the output to <out_file>. Specifying "-" as <out_file> will redirect to stdout. + --output-dir <out_directory> + An optional directory to which to store objects. + By default, all objects are recreated in the current + directory. --object-threads <threads> Number of threads to use for downloading objects. Default is 10. @@ -205,6 +218,14 @@ def st_download(parser, args, output_manager): 'download, stream the output to <out_file>. ' 'Specifying "-" as <out_file> will redirect to stdout.') parser.add_option( + '-D', '--output-dir', dest='out_directory', + help='An optional directory to which to store objects. ' + 'By default, all objects are recreated in the current directory.') + parser.add_option( + '-r', '--remove-prefix', action='store_true', dest='remove_prefix', + default=False, help='An optional flag for --prefix <prefix>, ' + 'use this option to download items without <prefix>.') + parser.add_option( '', '--object-threads', type=int, default=10, help='Number of threads to use for downloading objects. ' 'Default is 10.') @@ -234,6 +255,12 @@ def st_download(parser, args, output_manager): if options.out_file and len(args) != 2: exit('-o option only allowed for single file downloads') + if not options.prefix: + options.remove_prefix = False + + if options.out_directory and len(args) == 2: + exit('Please use -o option for single file downloads and renames') + if (not args and not options.yes_all) or (args and options.yes_all): output_manager.error('Usage: %s download %s\n%s', BASENAME, st_download_options, st_download_help) @@ -909,6 +936,46 @@ def st_capabilities(parser, args, output_manager): st_info = st_capabilities +st_auth_help = ''' +Display auth related authentication variables in shell friendly format. + + Commands to run to export storage url and auth token into + OS_STORAGE_URL and OS_AUTH_TOKEN: + + swift auth + + Commands to append to a runcom file (e.g. ~/.bashrc, /etc/profile) for + automatic authentication: + + swift auth -v -U test:tester -K testing \ + -A http://localhost:8080/auth/v1.0 + +'''.strip('\n') + + +def st_auth(parser, args, thread_manager): + (options, args) = parse_args(parser, args) + _opts = vars(options) + if options.verbose > 1: + if options.auth_version in ('1', '1.0'): + print('export ST_AUTH=%s' % sh_quote(options.auth)) + print('export ST_USER=%s' % sh_quote(options.user)) + print('export ST_KEY=%s' % sh_quote(options.key)) + else: + print('export OS_IDENTITY_API_VERSION=%s' % sh_quote( + options.auth_version)) + print('export OS_AUTH_VERSION=%s' % sh_quote(options.auth_version)) + print('export OS_AUTH_URL=%s' % sh_quote(options.auth)) + for k, v in sorted(_opts.items()): + if v and k.startswith('os_') and \ + k not in ('os_auth_url', 'os_options'): + print('export %s=%s' % (k.upper(), sh_quote(v))) + else: + conn = get_conn(_opts) + url, token = conn.get_auth() + print('export OS_STORAGE_URL=%s' % sh_quote(url)) + print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) + st_tempurl_options = '<method> <seconds> <path> <key>' @@ -917,13 +984,13 @@ st_tempurl_help = ''' Generates a temporary URL for a Swift object. Positional arguments: - [method] An HTTP method to allow for this temporary URL. + <method> An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'. - [seconds] The amount of time in seconds the temporary URL will + <seconds> The amount of time in seconds the temporary URL will be valid for. - [path] The full path to the Swift object. Example: + <path> The full path to the Swift object. Example: /v1/AUTH_account/c/o. - [key] The secret temporary URL key set on the Swift cluster. + <key> The secret temporary URL key set on the Swift cluster. To set a key, run \'swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' '''.strip('\n') @@ -1079,7 +1146,8 @@ Positional arguments: or object. upload Uploads files or directories to the given container. capabilities List cluster capabilities. - tempurl Create a temporary URL + tempurl Create a temporary URL. + auth Display auth related environment variables. Examples: %%prog download --help diff --git a/test-requirements.txt b/test-requirements.txt index 3c80411..909cb04 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,8 @@ hacking>=0.10.0,<0.11 coverage>=3.6 discover -mock>=1.0 +mock>=1.0;python_version!='2.6' +mock==1.0.1;python_version=='2.6' oslosphinx python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index ee1cbb6..339aca1 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1029,6 +1029,17 @@ class TestServiceUpload(testtools.TestCase): class TestServiceDownload(testtools.TestCase): + def setUp(self): + super(TestServiceDownload, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + self.opts['no_download'] = True + self.obj_content = b'c' * 10 + self.obj_etag = md5(self.obj_content).hexdigest() + self.obj_len = len(self.obj_content) + + def _readbody(self): + yield self.obj_content + def _assertDictEqual(self, a, b, m=None): # assertDictEqual is not available in py2.6 so use a shallow check # instead @@ -1045,6 +1056,103 @@ class TestServiceDownload(testtools.TestCase): self.assertIn(k, b, m) self.assertEqual(b[k], v, m) + def test_download(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = service._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['out_directory'] = 'temp_dir' + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'temp_dir/example/test') + + def test_download_with_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example/' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_remove_prefix_and_remove_slashes(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir_and_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['out_directory'] = 'new/dir' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'new/dir/test') + def test_download_object_job_skip_identical(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) @@ -1077,6 +1185,9 @@ class TestServiceDownload(testtools.TestCase): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1129,6 +1240,9 @@ class TestServiceDownload(testtools.TestCase): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1207,6 +1321,9 @@ class TestServiceDownload(testtools.TestCase): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1268,6 +1385,9 @@ class TestServiceDownload(testtools.TestCase): 'auth_end_time': mock_conn.auth_end_time, } + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True s = SwiftService() with mock.patch('swiftclient.service.time', side_effect=range(3)): with mock.patch('swiftclient.service.get_conn', @@ -1276,11 +1396,7 @@ class TestServiceDownload(testtools.TestCase): conn=mock_conn, container='test_c', obj='test_o', - options={'out_file': f.name, - 'header': {}, - 'no_download': True, - 'yes_all': False, - 'skip_identical': True}) + options=options) self._assertDictEqual(r, expected_r) @@ -1360,6 +1476,9 @@ class TestServiceDownload(testtools.TestCase): 'auth_end_time': mock_conn.auth_end_time, } + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True s = SwiftService() with mock.patch('swiftclient.service.time', side_effect=range(3)): with mock.patch('swiftclient.service.get_conn', @@ -1368,11 +1487,7 @@ class TestServiceDownload(testtools.TestCase): conn=mock_conn, container='test_c', obj='test_o', - options={'out_file': f.name, - 'header': {}, - 'no_download': True, - 'yes_all': False, - 'skip_identical': True}) + options=options) self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 9f97990..8384f7a 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -20,8 +20,10 @@ import mock import os import tempfile import unittest +import textwrap from testtools import ExpectedException + import six import swiftclient @@ -47,6 +49,11 @@ mocked_os_environ = { 'ST_USER': 'test:tester', 'ST_KEY': 'testing' } +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -1587,6 +1594,101 @@ class TestAuth(MockHttpTest): }), ]) + def test_auth(self): + headers = { + 'x-auth-token': 'AUTH_tk5b6b12', + 'x-storage-url': 'https://swift.storage.example.com/v1/AUTH_test', + } + mock_resp = self.fake_http_connection(200, headers=headers) + with mock.patch('swiftclient.client.http_connection', new=mock_resp): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'testing', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test + export OS_AUTH_TOKEN=AUTH_tk5b6b12 + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose(self): + with mock.patch('swiftclient.client.http_connection') as mock_conn: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'te$tin&', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export ST_AUTH=https://swift.storage.example.com/auth/v1.0 + export ST_USER=test:tester + export ST_KEY='te$tin&' + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_conn.mock_calls) + + def test_auth_v2(self): + os_options = {'tenant_name': 'demo'} + with mock.patch('swiftclient.client.get_auth_keystone', + new=fake_get_auth_keystone(os_options)): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', 'admin', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=http://url/ + export OS_AUTH_TOKEN=token + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose_v2(self): + with mock.patch('swiftclient.client.get_auth_keystone') \ + as mock_keystone: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', '$eKr3t', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_IDENTITY_API_VERSION=2.0 + export OS_AUTH_VERSION=2.0 + export OS_AUTH_URL=https://keystone.example.com/v2.0/ + export OS_PASSWORD='$eKr3t' + export OS_TENANT_NAME=demo + export OS_USERNAME=demo + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_keystone.mock_calls) + class TestCrossAccountObjectAccess(TestBase, MockHttpTest): """ diff --git a/tests/unit/utils.py b/tests/unit/utils.py index aced173..4f7c8ec 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -38,8 +38,10 @@ def fake_get_auth_keystone(expected_os_options=None, exc=None, if exc: raise exc('test') # TODO: some way to require auth_url, user and key? - if expected_os_options and actual_os_options != expected_os_options: - return "", None + if expected_os_options: + for key, value in actual_os_options.items(): + if value and value != expected_os_options.get(key): + return "", None if 'required_kwargs' in kwargs: for k, v in kwargs['required_kwargs'].items(): if v != actual_kwargs.get(k): |