diff options
| -rwxr-xr-x | bin/swift-dispersion-populate | 14 | ||||
| -rwxr-xr-x | bin/swift-dispersion-report | 12 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | swift/common/internal_client.py | 107 | ||||
| -rw-r--r-- | swift/container/sync.py | 4 | ||||
| -rw-r--r-- | test-requirements.txt | 1 | ||||
| -rw-r--r-- | test/unit/common/test_internal_client.py | 107 |
7 files changed, 231 insertions, 15 deletions
diff --git a/bin/swift-dispersion-populate b/bin/swift-dispersion-populate index 351af6309..a475e40e4 100755 --- a/bin/swift-dispersion-populate +++ b/bin/swift-dispersion-populate @@ -24,7 +24,11 @@ from time import time from eventlet import GreenPool, patcher, sleep from eventlet.pools import Pool -from swiftclient import Connection, get_auth +try: + from swiftclient import get_auth +except ImportError: + from swift.common.internal_client import get_auth +from swift.common.internal_client import SimpleClient from swift.common.ring import Ring from swift.common.utils import compute_eta, get_time_units, config_true_value @@ -133,12 +137,8 @@ Usage: %%prog [options] [conf_file] insecure=insecure) account = url.rsplit('/', 1)[1] connpool = Pool(max_size=concurrency) - connpool.create = lambda: Connection(conf['auth_url'], - conf['auth_user'], conf['auth_key'], - retries=retries, - preauthurl=url, preauthtoken=token, - os_options=os_options, - insecure=insecure) + connpool.create = lambda: SimpleClient( + url=url, token=token, retries=retries) if container_populate: container_ring = Ring(swift_dir, ring_name='container') diff --git a/bin/swift-dispersion-report b/bin/swift-dispersion-report index aabea6c50..34f239c87 100755 --- a/bin/swift-dispersion-report +++ b/bin/swift-dispersion-report @@ -28,7 +28,11 @@ from eventlet import GreenPool, hubs, patcher, Timeout from eventlet.pools import Pool from swift.common import direct_client -from swiftclient import Connection, get_auth +try: + from swiftclient import get_auth +except ImportError: + from swift.common.internal_client import get_auth +from swift.common.internal_client import SimpleClient from swift.common.ring import Ring from swift.common.exceptions import ClientException from swift.common.utils import compute_eta, get_time_units, config_true_value @@ -356,10 +360,8 @@ Usage: %%prog [options] [conf_file] insecure=insecure) account = url.rsplit('/', 1)[1] connpool = Pool(max_size=concurrency) - connpool.create = lambda: Connection( - conf['auth_url'], conf['auth_user'], conf['auth_key'], retries=retries, - preauthurl=url, preauthtoken=token, os_options=os_options, - insecure=insecure) + connpool.create = lambda: SimpleClient( + url=url, token=token, retries=retries) container_ring = Ring(swift_dir, ring_name='container') object_ring = Ring(swift_dir, ring_name='object') diff --git a/requirements.txt b/requirements.txt index 113986693..bbac51a61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,3 @@ netifaces>=0.5 pastedeploy>=1.3.3 simplejson>=2.0.9 xattr>=0.4 -python-swiftclient diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index f6a22c818..8394706a0 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -14,12 +14,14 @@ # limitations under the License. from eventlet import sleep, Timeout +from eventlet.green import httplib, socket, urllib2 import json from paste.deploy import loadapp import struct from sys import exc_info import zlib from swift import gettext_ as _ +import urlparse from zlib import compressobj from swift.common.utils import quote @@ -675,3 +677,108 @@ class InternalClient(object): headers['Transfer-Encoding'] = 'chunked' path = self.make_path(account, container, obj) self.make_request('PUT', path, headers, (2,), fobj) + + +def get_auth(url, user, key, auth_version='1.0', **kwargs): + if auth_version != '1.0': + exit('ERROR: swiftclient missing, only auth v1.0 supported') + req = urllib2.Request(url) + req.add_header('X-Auth-User', user) + req.add_header('X-Auth-Key', key) + conn = urllib2.urlopen(req) + headers = conn.info() + return ( + headers.getheader('X-Storage-Url'), + headers.getheader('X-Auth-Token')) + + +class SimpleClient(object): + """ + Simple client that is used in bin/swift-dispersion-* and container sync + """ + def __init__(self, url=None, token=None, starting_backoff=1, + max_backoff=5, retries=5): + self.url = url + self.token = token + self.attempts = 0 + self.starting_backoff = starting_backoff + self.max_backoff = max_backoff + self.retries = retries + + def base_request(self, method, container=None, name=None, prefix=None, + headers={}, proxy=None, contents=None, full_listing=None): + # Common request method + url = self.url + + if self.token: + headers['X-Auth-Token'] = self.token + + if container: + url = '%s/%s' % (url.rstrip('/'), quote(container)) + + if name: + url = '%s/%s' % (url.rstrip('/'), quote(name)) + + url += '?format=json' + + if prefix: + url += '&prefix=%s' % prefix + + if proxy: + proxy = urlparse.urlparse(proxy) + proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc}) + opener = urllib2.build_opener(proxy) + urllib2.install_opener(opener) + + req = urllib2.Request(url, headers=headers, data=contents) + req.get_method = lambda: method + urllib2.urlopen(req) + conn = urllib2.urlopen(req) + body = conn.read() + try: + body_data = json.loads(body) + except ValueError: + body_data = None + return [None, body_data] + + def retry_request(self, method, **kwargs): + self.attempts = 0 + backoff = self.starting_backoff + while self.attempts <= self.retries: + self.attempts += 1 + try: + return self.base_request(method, **kwargs) + except (socket.error, httplib.HTTPException, urllib2.URLError): + if self.attempts > self.retries: + raise + sleep(backoff) + backoff = min(backoff * 2, self.max_backoff) + + def get_account(self, *args, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('GET', **kwargs) + + def put_container(self, container, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('PUT', container=container, **kwargs) + + def get_container(self, container, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('GET', container=container, **kwargs) + + def put_object(self, container, name, contents, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('PUT', container=container, name=name, + contents=contents.read(), **kwargs) + + +def put_object(url, **kwargs): + """For usage with container sync """ + client = SimpleClient(url=url) + client.retry_request('PUT', **kwargs) + + +def delete_object(url, **kwargs): + """For usage with container sync """ + client = SimpleClient(url=url) + client.retry_request('DELETE', **kwargs) diff --git a/swift/container/sync.py b/swift/container/sync.py index 41302bfb7..87ca387b7 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -24,15 +24,15 @@ from eventlet import sleep, Timeout import swift.common.db from swift.container import server as container_server -from swiftclient import delete_object, put_object, quote from swift.container.backend import ContainerBroker from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.direct_client import direct_get_object +from swift.common.internal_client import delete_object, put_object from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import audit_location_generator, get_logger, \ hash_path, config_true_value, validate_sync_to, whataremyips, \ - FileLikeIter, urlparse + FileLikeIter, urlparse, quote from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND diff --git a/test-requirements.txt b/test-requirements.txt index 8db26120a..7036e8d19 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,3 +7,4 @@ openstack.nose_plugin nosehtmloutput sphinx>=1.1.2,<1.2 mock>=0.8.0 +python-swiftclient diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index 6c7e6272e..4772688d6 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -14,11 +14,13 @@ # limitations under the License. import json +import mock from StringIO import StringIO import unittest from urllib import quote import zlib +from eventlet.green import urllib2 from swift.common import internal_client @@ -919,5 +921,110 @@ class TestInternalClient(unittest.TestCase): client.upload_object(fobj, account, container, obj, headers) self.assertEquals(1, client.make_request_called) + +class TestGetAuth(unittest.TestCase): + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_ok(self, request, urlopen): + def getheader(name): + d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'} + return d.get(name) + urlopen.return_value.info.return_value.getheader = getheader + + url, token = internal_client.get_auth( + 'http://127.0.0.1', 'user', 'key') + + self.assertEqual(url, "url") + self.assertEqual(token, "token") + request.assert_called_with('http://127.0.0.1') + request.return_value.add_header.assert_any_call('X-Auth-User', 'user') + request.return_value.add_header.assert_any_call('X-Auth-Key', 'key') + + def test_invalid_version(self): + self.assertRaises(SystemExit, internal_client.get_auth, + 'http://127.0.0.1', 'user', 'key', auth_version=2.0) + + +class TestSimpleClient(unittest.TestCase): + + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_get(self, request, urlopen): + # basic GET request, only url as kwarg + request.return_value.get_type.return_value = "http" + urlopen.return_value.read.return_value = '' + sc = internal_client.SimpleClient(url='http://127.0.0.1') + retval = sc.retry_request('GET') + request.assert_called_with('http://127.0.0.1?format=json', + headers={}, + data=None) + self.assertEqual([None, None], retval) + self.assertEqual('GET', request.return_value.get_method()) + + # Check if JSON is decoded + urlopen.return_value.read.return_value = '{}' + retval = sc.retry_request('GET') + self.assertEqual([None, {}], retval) + + # same as above, now with token + sc = internal_client.SimpleClient(url='http://127.0.0.1', + token='token') + retval = sc.retry_request('GET') + request.assert_called_with('http://127.0.0.1?format=json', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + # same as above, now with prefix + sc = internal_client.SimpleClient(url='http://127.0.0.1', + token='token') + retval = sc.retry_request('GET', prefix="pre_") + request.assert_called_with('http://127.0.0.1?format=json&prefix=pre_', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + # same as above, now with container name + retval = sc.retry_request('GET', container='cont') + request.assert_called_with('http://127.0.0.1/cont?format=json', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + # same as above, now with object name + retval = sc.retry_request('GET', container='cont', name='obj') + request.assert_called_with('http://127.0.0.1/cont/obj?format=json', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_get_with_retries_all_failed(self, request, urlopen): + # Simulate a failing request, ensure retries done + request.return_value.get_type.return_value = "http" + request.side_effect = urllib2.URLError('') + urlopen.return_value.read.return_value = '' + sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1) + self.assertRaises(urllib2.URLError, sc.retry_request, 'GET') + self.assertEqual(request.call_count, 2) + + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_get_with_retries(self, request, urlopen): + # First request fails, retry successful + request.return_value.get_type.return_value = "http" + urlopen.return_value.read.return_value = '' + req = urllib2.Request('http://127.0.0.1', method='GET') + request.side_effect = [urllib2.URLError(''), req] + sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1) + + retval = sc.retry_request('GET') + self.assertEqual(request.call_count, 3) + request.assert_called_with('http://127.0.0.1?format=json', data=None, + headers={'X-Auth-Token': 'token'}) + self.assertEqual([None, None], retval) + + if __name__ == '__main__': unittest.main() |