Remove swiftclient dependency

Removes the requirement for swiftclient in swift-dispersion-report
and swift-dispersion-populate. To prevent a dependency on
keystoneclient and to avoid reinventing the wheel with an internal
keystoneclient, authentication with keystone is only supported if
swiftclient is available. If not, only auth v1 is supported.

The dependency in swift/container/sync.py has also been removed.

Implements: blueprint remove-swiftclient-dependency

Change-Id: I6ec3b3c85a67b9ab6eb04b90ffc16daf1600e8a7

7 files changed, 231 insertions, 15 deletions

diff --git a/bin/swift-dispersion-populate b/bin/swift-dispersion-populate
index 351af6309..a475e40e4 100755
--- a/bin/swift-dispersion-populate
+++ b/bin/swift-dispersion-populate
@@ -24,7 +24,11 @@ from time import time
 from eventlet import GreenPool, patcher, sleep
 from eventlet.pools import Pool
 
-from swiftclient import Connection, get_auth
+try:
+    from swiftclient import get_auth
+except ImportError:
+    from swift.common.internal_client import get_auth
+from swift.common.internal_client import SimpleClient
 from swift.common.ring import Ring
 from swift.common.utils import compute_eta, get_time_units, config_true_value
 
@@ -133,12 +137,8 @@ Usage: %%prog [options] [conf_file]
                           insecure=insecure)
     account = url.rsplit('/', 1)[1]
     connpool = Pool(max_size=concurrency)
-    connpool.create = lambda: Connection(conf['auth_url'],
-                                         conf['auth_user'], conf['auth_key'],
-                                         retries=retries,
-                                         preauthurl=url, preauthtoken=token,
-                                         os_options=os_options,
-                                         insecure=insecure)
+    connpool.create = lambda: SimpleClient(
+        url=url, token=token, retries=retries)
 
     if container_populate:
         container_ring = Ring(swift_dir, ring_name='container')
diff --git a/bin/swift-dispersion-report b/bin/swift-dispersion-report
index aabea6c50..34f239c87 100755
--- a/bin/swift-dispersion-report
+++ b/bin/swift-dispersion-report
@@ -28,7 +28,11 @@ from eventlet import GreenPool, hubs, patcher, Timeout
 from eventlet.pools import Pool
 
 from swift.common import direct_client
-from swiftclient import Connection, get_auth
+try:
+    from swiftclient import get_auth
+except ImportError:
+    from swift.common.internal_client import get_auth
+from swift.common.internal_client import SimpleClient
 from swift.common.ring import Ring
 from swift.common.exceptions import ClientException
 from swift.common.utils import compute_eta, get_time_units, config_true_value
@@ -356,10 +360,8 @@ Usage: %%prog [options] [conf_file]
                           insecure=insecure)
     account = url.rsplit('/', 1)[1]
     connpool = Pool(max_size=concurrency)
-    connpool.create = lambda: Connection(
-        conf['auth_url'], conf['auth_user'], conf['auth_key'], retries=retries,
-        preauthurl=url, preauthtoken=token, os_options=os_options,
-        insecure=insecure)
+    connpool.create = lambda: SimpleClient(
+        url=url, token=token, retries=retries)
 
     container_ring = Ring(swift_dir, ring_name='container')
     object_ring = Ring(swift_dir, ring_name='object')
diff --git a/requirements.txt b/requirements.txt
index 113986693..bbac51a61 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,3 @@ netifaces>=0.5
 pastedeploy>=1.3.3
 simplejson>=2.0.9
 xattr>=0.4
-python-swiftclient
diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py
index f6a22c818..8394706a0 100644
--- a/swift/common/internal_client.py
+++ b/swift/common/internal_client.py
@@ -14,12 +14,14 @@
 # limitations under the License.
 
 from eventlet import sleep, Timeout
+from eventlet.green import httplib, socket, urllib2
 import json
 from paste.deploy import loadapp
 import struct
 from sys import exc_info
 import zlib
 from swift import gettext_ as _
+import urlparse
 from zlib import compressobj
 
 from swift.common.utils import quote
@@ -675,3 +677,108 @@ class InternalClient(object):
         headers['Transfer-Encoding'] = 'chunked'
         path = self.make_path(account, container, obj)
         self.make_request('PUT', path, headers, (2,), fobj)
+
+
+def get_auth(url, user, key, auth_version='1.0', **kwargs):
+    if auth_version != '1.0':
+        exit('ERROR: swiftclient missing, only auth v1.0 supported')
+    req = urllib2.Request(url)
+    req.add_header('X-Auth-User', user)
+    req.add_header('X-Auth-Key', key)
+    conn = urllib2.urlopen(req)
+    headers = conn.info()
+    return (
+        headers.getheader('X-Storage-Url'),
+        headers.getheader('X-Auth-Token'))
+
+
+class SimpleClient(object):
+    """
+    Simple client that is used in bin/swift-dispersion-* and container sync
+    """
+    def __init__(self, url=None, token=None, starting_backoff=1,
+                 max_backoff=5, retries=5):
+        self.url = url
+        self.token = token
+        self.attempts = 0
+        self.starting_backoff = starting_backoff
+        self.max_backoff = max_backoff
+        self.retries = retries
+
+    def base_request(self, method, container=None, name=None, prefix=None,
+                     headers={}, proxy=None, contents=None, full_listing=None):
+        # Common request method
+        url = self.url
+
+        if self.token:
+            headers['X-Auth-Token'] = self.token
+
+        if container:
+            url = '%s/%s' % (url.rstrip('/'), quote(container))
+
+        if name:
+            url = '%s/%s' % (url.rstrip('/'), quote(name))
+
+        url += '?format=json'
+
+        if prefix:
+            url += '&prefix=%s' % prefix
+
+        if proxy:
+            proxy = urlparse.urlparse(proxy)
+            proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc})
+            opener = urllib2.build_opener(proxy)
+            urllib2.install_opener(opener)
+
+        req = urllib2.Request(url, headers=headers, data=contents)
+        req.get_method = lambda: method
+        urllib2.urlopen(req)
+        conn = urllib2.urlopen(req)
+        body = conn.read()
+        try:
+            body_data = json.loads(body)
+        except ValueError:
+            body_data = None
+        return [None, body_data]
+
+    def retry_request(self, method, **kwargs):
+        self.attempts = 0
+        backoff = self.starting_backoff
+        while self.attempts <= self.retries:
+            self.attempts += 1
+            try:
+                return self.base_request(method, **kwargs)
+            except (socket.error, httplib.HTTPException, urllib2.URLError):
+                if self.attempts > self.retries:
+                    raise
+            sleep(backoff)
+            backoff = min(backoff * 2, self.max_backoff)
+
+    def get_account(self, *args, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('GET', **kwargs)
+
+    def put_container(self, container, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('PUT', container=container, **kwargs)
+
+    def get_container(self, container, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('GET', container=container, **kwargs)
+
+    def put_object(self, container, name, contents, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('PUT', container=container, name=name,
+                                  contents=contents.read(), **kwargs)
+
+
+def put_object(url, **kwargs):
+    """For usage with container sync """
+    client = SimpleClient(url=url)
+    client.retry_request('PUT', **kwargs)
+
+
+def delete_object(url, **kwargs):
+    """For usage with container sync """
+    client = SimpleClient(url=url)
+    client.retry_request('DELETE', **kwargs)
diff --git a/swift/container/sync.py b/swift/container/sync.py
index 41302bfb7..87ca387b7 100644
--- a/swift/container/sync.py
+++ b/swift/container/sync.py
@@ -24,15 +24,15 @@ from eventlet import sleep, Timeout
 
 import swift.common.db
 from swift.container import server as container_server
-from swiftclient import delete_object, put_object, quote
 from swift.container.backend import ContainerBroker
 from swift.common.container_sync_realms import ContainerSyncRealms
 from swift.common.direct_client import direct_get_object
+from swift.common.internal_client import delete_object, put_object
 from swift.common.exceptions import ClientException
 from swift.common.ring import Ring
 from swift.common.utils import audit_location_generator, get_logger, \
     hash_path, config_true_value, validate_sync_to, whataremyips, \
-    FileLikeIter, urlparse
+    FileLikeIter, urlparse, quote
 from swift.common.daemon import Daemon
 from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
 
diff --git a/test-requirements.txt b/test-requirements.txt
index 8db26120a..7036e8d19 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -7,3 +7,4 @@ openstack.nose_plugin
 nosehtmloutput
 sphinx>=1.1.2,<1.2
 mock>=0.8.0
+python-swiftclient
diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py
index 6c7e6272e..4772688d6 100644
--- a/test/unit/common/test_internal_client.py
+++ b/test/unit/common/test_internal_client.py
@@ -14,11 +14,13 @@
 # limitations under the License.
 
 import json
+import mock
 from StringIO import StringIO
 import unittest
 from urllib import quote
 import zlib
 
+from eventlet.green import urllib2
 from swift.common import internal_client
 
 
@@ -919,5 +921,110 @@ class TestInternalClient(unittest.TestCase):
         client.upload_object(fobj, account, container, obj, headers)
         self.assertEquals(1, client.make_request_called)
 
+
+class TestGetAuth(unittest.TestCase):
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_ok(self, request, urlopen):
+        def getheader(name):
+            d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'}
+            return d.get(name)
+        urlopen.return_value.info.return_value.getheader = getheader
+
+        url, token = internal_client.get_auth(
+            'http://127.0.0.1', 'user', 'key')
+
+        self.assertEqual(url, "url")
+        self.assertEqual(token, "token")
+        request.assert_called_with('http://127.0.0.1')
+        request.return_value.add_header.assert_any_call('X-Auth-User', 'user')
+        request.return_value.add_header.assert_any_call('X-Auth-Key', 'key')
+
+    def test_invalid_version(self):
+        self.assertRaises(SystemExit, internal_client.get_auth,
+                          'http://127.0.0.1', 'user', 'key', auth_version=2.0)
+
+
+class TestSimpleClient(unittest.TestCase):
+
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_get(self, request, urlopen):
+        # basic GET request, only url as kwarg
+        request.return_value.get_type.return_value = "http"
+        urlopen.return_value.read.return_value = ''
+        sc = internal_client.SimpleClient(url='http://127.0.0.1')
+        retval = sc.retry_request('GET')
+        request.assert_called_with('http://127.0.0.1?format=json',
+                                   headers={},
+                                   data=None)
+        self.assertEqual([None, None], retval)
+        self.assertEqual('GET', request.return_value.get_method())
+
+        # Check if JSON is decoded
+        urlopen.return_value.read.return_value = '{}'
+        retval = sc.retry_request('GET')
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with token
+        sc = internal_client.SimpleClient(url='http://127.0.0.1',
+                                          token='token')
+        retval = sc.retry_request('GET')
+        request.assert_called_with('http://127.0.0.1?format=json',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with prefix
+        sc = internal_client.SimpleClient(url='http://127.0.0.1',
+                                          token='token')
+        retval = sc.retry_request('GET', prefix="pre_")
+        request.assert_called_with('http://127.0.0.1?format=json&prefix=pre_',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with container name
+        retval = sc.retry_request('GET', container='cont')
+        request.assert_called_with('http://127.0.0.1/cont?format=json',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with object name
+        retval = sc.retry_request('GET', container='cont', name='obj')
+        request.assert_called_with('http://127.0.0.1/cont/obj?format=json',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_get_with_retries_all_failed(self, request, urlopen):
+        # Simulate a failing request, ensure retries done
+        request.return_value.get_type.return_value = "http"
+        request.side_effect = urllib2.URLError('')
+        urlopen.return_value.read.return_value = ''
+        sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
+        self.assertRaises(urllib2.URLError, sc.retry_request, 'GET')
+        self.assertEqual(request.call_count, 2)
+
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_get_with_retries(self, request, urlopen):
+        # First request fails, retry successful
+        request.return_value.get_type.return_value = "http"
+        urlopen.return_value.read.return_value = ''
+        req = urllib2.Request('http://127.0.0.1', method='GET')
+        request.side_effect = [urllib2.URLError(''), req]
+        sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
+
+        retval = sc.retry_request('GET')
+        self.assertEqual(request.call_count, 3)
+        request.assert_called_with('http://127.0.0.1?format=json', data=None,
+                                   headers={'X-Auth-Token': 'token'})
+        self.assertEqual([None, None], retval)
+
+
 if __name__ == '__main__':
     unittest.main()