Enable usage of proxies defined in environment (http(s)_proxy).

As far as proxies usage is concerned, keystone-client API and swift-client API behave differently because the former uses python Request library while the latter uses raw httplib. As a result, Keystone authentication honors environment variables http_proxy, https_proxy and no_proxy while Swift doesn't.

This patch, which code is mainly borrowed from Python Requests, makes Swift
data connections and Swift authentication connections behaving homogeneously.

Change-Id: Ic8a0089c35c458d7ed96e572e22429014298fe4c

3 files changed, 138 insertions, 2 deletions

diff --git a/swiftclient/client.py b/swiftclient/client.py
index 8dc4b71..edd0913 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -29,6 +29,7 @@ from httplib import HTTPException, HTTPConnection, HTTPSConnection
 from time import sleep
 
 from swiftclient.exceptions import ClientException, InvalidHeadersException
+from swiftclient.utils import get_environ_proxies
 
 try:
     from swiftclient.https_connection import HTTPSConnectionNoSSLComp
@@ -138,8 +139,13 @@ def http_connection(url, proxy=None, ssl_compression=True):
     """
     url = encode_utf8(url)
     parsed = urlparse(url)
-    proxy_parsed = urlparse(proxy) if proxy else None
-    host = proxy_parsed if proxy else parsed.netloc
+    if proxy:
+        proxy_parsed = urlparse(proxy)
+    else:
+        proxies = get_environ_proxies(parsed.netloc)
+        proxy = proxies.get(parsed.scheme, None)
+        proxy_parsed = urlparse(proxy) if proxy else None
+    host = proxy_parsed.netloc if proxy else parsed.netloc
     if parsed.scheme == 'http':
         conn = HTTPConnection(host)
     elif parsed.scheme == 'https':
diff --git a/swiftclient/utils.py b/swiftclient/utils.py
index a038dcc..a74eada 100644
--- a/swiftclient/utils.py
+++ b/swiftclient/utils.py
@@ -14,6 +14,23 @@
 # limitations under the License.
 """Miscellaneous utility functions for use with Swift."""
 
+import sys
+import os
+
+_ver = sys.version_info
+
+#: Python 2.x?
+is_py2 = (_ver[0] == 2)
+
+#: Python 3.x?
+is_py3 = (_ver[0] == 3)
+
+if is_py2:
+    from urllib import getproxies, proxy_bypass
+elif is_py3:
+    from urllib.request import getproxies, proxy_bypass
+
+
 TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
 
 
@@ -55,3 +72,36 @@ def prt_bytes(bytes, human_flag):
         bytes = '%12s' % bytes
 
     return(bytes)
+
+
+# get_environ_proxies function, borrowed from python Requests
+# (www.python-requests.org)
+def get_environ_proxies(netloc):
+    """Return a dict of environment proxies."""
+
+    get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
+
+    # First check whether no_proxy is defined. If it is, check that the URL
+    # we're getting isn't in the no_proxy list.
+    no_proxy = get_proxy('no_proxy')
+
+    if no_proxy:
+        # We need to check whether we match here. We need to see if we match
+        # the end of the netloc, both with and without the port.
+        no_proxy = no_proxy.replace(' ', '').split(',')
+
+        for host in no_proxy:
+            if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
+                # The URL does match something in no_proxy, so we don't want
+                # to apply the proxies on this URL.
+                return {}
+
+    # If the system proxy settings indicate that this URL should be bypassed,
+    # don't proxy.
+    if proxy_bypass(netloc):
+        return {}
+
+    # If we get here, we either didn't have no_proxy set or we're not going
+    # anywhere that no_proxy applies to, and the system settings don't require
+    # bypassing the proxy for the current URL.
+    return getproxies()
diff --git a/tests/test_utils.py b/tests/test_utils.py
index b47d231..e64d29d 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import testtools
+import os
 
 from swiftclient import utils as u
 
@@ -117,3 +118,82 @@ class TestPrtBytes(testtools.TestCase):
     def test_overflow(self):
         bytes_ = 2 ** 90
         self.assertEquals('1024Y', u.prt_bytes(bytes_, True).lstrip())
+
+
+class TestGetEnvironProxy(testtools.TestCase):
+
+    ENV_VARS = ('http_proxy', 'https_proxy', 'no_proxy',
+                'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY')
+
+    def setUp(self):
+        self.proxy_dict = {}
+        super(TestGetEnvironProxy, self).setUp()
+        for proxy_s in TestGetEnvironProxy.ENV_VARS:
+            # Save old env value
+            self.proxy_dict[proxy_s] = os.environ.get(proxy_s, None)
+
+    def tearDown(self):
+        super(TestGetEnvironProxy, self).tearDown()
+        for proxy_s in TestGetEnvironProxy.ENV_VARS:
+            if self.proxy_dict[proxy_s]:
+                os.environ[proxy_s] = self.proxy_dict[proxy_s]
+            elif os.environ.get(proxy_s):
+                del os.environ[proxy_s]
+
+    def setup_env(self, new_env={}):
+        for proxy_s in TestGetEnvironProxy.ENV_VARS:
+            # Set new env value
+            if new_env.get(proxy_s):
+                os.environ[proxy_s] = new_env.get(proxy_s)
+            elif os.environ.get(proxy_s):
+                del os.environ[proxy_s]
+
+    def test_http_proxy(self):
+        self.setup_env({'http_proxy': 'http://proxy.tests.com:8080'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(proxy_dict['http'], 'http://proxy.tests.com:8080')
+        self.assertEquals(proxy_dict.get('https'), None)
+        self.assertEquals(len(proxy_dict), 1)
+        self.setup_env({'HTTP_PROXY': 'http://proxy.tests.com:8080'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(proxy_dict['http'], 'http://proxy.tests.com:8080')
+        self.assertEquals(proxy_dict.get('https'), None)
+        self.assertEquals(len(proxy_dict), 1)
+
+    def test_https_proxy(self):
+        self.setup_env({'https_proxy': 'http://proxy.tests.com:8080'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(proxy_dict['https'], 'http://proxy.tests.com:8080')
+        self.assertEquals(proxy_dict.get('http'), None)
+        self.assertEquals(len(proxy_dict), 1)
+        self.setup_env({'HTTPS_PROXY': 'http://proxy.tests.com:8080'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(proxy_dict['https'], 'http://proxy.tests.com:8080')
+        self.assertEquals(proxy_dict.get('http'), None)
+        self.assertEquals(len(proxy_dict), 1)
+
+    def test_http_https_proxy(self):
+        self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
+                        'https_proxy': 'http://proxy2.tests.com:8082'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(proxy_dict['http'], 'http://proxy1.tests.com:8081')
+        self.assertEquals(proxy_dict['https'], 'http://proxy2.tests.com:8082')
+        self.assertEquals(len(proxy_dict), 2)
+        self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
+                        'HTTPS_PROXY': 'http://proxy2.tests.com:8082'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(proxy_dict['http'], 'http://proxy1.tests.com:8081')
+        self.assertEquals(proxy_dict['https'], 'http://proxy2.tests.com:8082')
+        self.assertEquals(len(proxy_dict), 2)
+
+    def test_proxy_exclusion(self):
+        self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
+                        'https_proxy': 'http://proxy2.tests.com:8082',
+                        'no_proxy': 'www.tests.com'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(len(proxy_dict), 0)
+        self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
+                        'HTTPS_PROXY': 'http://proxy2.tests.com:8082',
+                        'NO_PROXY': 'www.tests.com'})
+        proxy_dict = u.get_environ_proxies('www.tests.com:81')
+        self.assertEquals(len(proxy_dict), 0)