diff options
Diffstat (limited to 'swiftclient')
| -rw-r--r-- | swiftclient/client.py | 204 | ||||
| -rw-r--r-- | swiftclient/https_connection.py | 95 | ||||
| -rw-r--r-- | swiftclient/utils.py | 50 |
3 files changed, 114 insertions, 235 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py index d23d467..ee80769 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -18,24 +18,18 @@ OpenStack Swift client library used internally """ import socket +import requests import sys import logging import warnings -from functools import wraps +from distutils.version import StrictVersion +from requests.exceptions import RequestException, SSLError from urllib import quote as _quote from urlparse import urlparse, urlunparse -from httplib import HTTPException, HTTPConnection, HTTPSConnection from time import sleep, time from swiftclient.exceptions import ClientException, InvalidHeadersException -from swiftclient.utils import get_environ_proxies - -try: - from swiftclient.https_connection import HTTPSConnectionNoSSLComp -except ImportError: - HTTPSConnectionNoSSLComp = HTTPSConnection - try: from logging import NullHandler @@ -50,6 +44,18 @@ except ImportError: def createLock(self): self.lock = None +# requests version 1.2.3 try to encode headers in ascii, preventing +# utf-8 encoded header to be 'prepared' +if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): + from requests.structures import CaseInsensitiveDict + + def prepare_unicode_headers(self, headers): + if headers: + self.headers = CaseInsensitiveDict(headers) + else: + self.headers = CaseInsensitiveDict() + requests.models.PreparedRequest.prepare_headers = prepare_unicode_headers + logger = logging.getLogger("swiftclient") logger.addHandler(NullHandler()) @@ -124,68 +130,93 @@ except ImportError: from json import loads as json_loads -def http_connection(url, proxy=None, ssl_compression=True): - """ - Make an HTTPConnection or HTTPSConnection +class HTTPConnection: + def __init__(self, url, proxy=None, cacert=None, insecure=False, + ssl_compression=False): + """ + Make an HTTPConnection or HTTPSConnection + + :param url: url to connect to + :param proxy: proxy to connect through, if any; None by default; str + of the format 'http://127.0.0.1:8888' to set one + :param cacert: A CA bundle file to use in verifying a TLS server + certificate. + :param insecure: Allow to access servers without checking SSL certs. + The server's certificate will not be verified. + :param ssl_compression: SSL compression should be disabled by default + and this setting is not usable as of now. The + parameter is kept for backward compatibility. + :raises ClientException: Unable to handle protocol scheme + """ + self.url = url + self.parsed_url = urlparse(url) + self.host = self.parsed_url.netloc + self.port = self.parsed_url.port + self.requests_args = {} + if self.parsed_url.scheme not in ('http', 'https'): + raise ClientException("Unsupported scheme") + self.requests_args['verify'] = not insecure + if cacert: + # verify requests parameter is used to pass the CA_BUNDLE file + # see: http://docs.python-requests.org/en/latest/user/advanced/ + self.requests_args['verify'] = cacert + if proxy: + proxy_parsed = urlparse(proxy) + if not proxy_parsed.scheme: + raise ClientException("Proxy's missing scheme") + self.requests_args['proxies'] = { + proxy_parsed.scheme: '%s://%s' % ( + proxy_parsed.scheme, proxy_parsed.netloc + ) + } + self.requests_args['stream'] = True + + def _request(self, *arg, **kwarg): + """ Final wrapper before requests call, to be patched in tests """ + return requests.request(*arg, **kwarg) + + def request(self, method, full_path, data=None, headers={}, files=None): + """ Encode url and header, then call requests.request """ + headers = dict((encode_utf8(x), encode_utf8(y)) for x, y in + headers.iteritems()) + url = encode_utf8("%s://%s%s" % ( + self.parsed_url.scheme, + self.parsed_url.netloc, + full_path)) + self.resp = self._request(method, url, headers=headers, data=data, + files=files, **self.requests_args) + return self.resp + + def putrequest(self, full_path, data=None, headers={}, files=None): + """ + Use python-requests files upload - :param url: url to connect to - :param proxy: proxy to connect through, if any; None by default; str of the - format 'http://127.0.0.1:8888' to set one - :param ssl_compression: Whether to enable compression at the SSL layer. - If set to 'False' and the pyOpenSSL library is - present an attempt to disable SSL compression - will be made. This may provide a performance - increase for https upload/download operations. - :returns: tuple of (parsed url, connection object) - :raises ClientException: Unable to handle protocol scheme - """ - url = encode_utf8(url) - parsed = urlparse(url) - if proxy: - proxy_parsed = urlparse(proxy) - else: - proxies = get_environ_proxies(parsed.netloc) - proxy = proxies.get(parsed.scheme, None) - proxy_parsed = urlparse(proxy) if proxy else None - host = proxy_parsed.netloc if proxy else parsed.netloc - if parsed.scheme == 'http': - conn = HTTPConnection(host) - elif parsed.scheme == 'https': - if ssl_compression is True: - conn = HTTPSConnection(host) - else: - conn = HTTPSConnectionNoSSLComp(host) - else: - raise ClientException('Cannot handle protocol scheme %s for url %s' % - (parsed.scheme, repr(url))) - - def putheader_wrapper(func): - - @wraps(func) - def putheader_escaped(key, value): - func(encode_utf8(key), encode_utf8(value)) - return putheader_escaped - conn.putheader = putheader_wrapper(conn.putheader) - - def request_wrapper(func): - - @wraps(func) - def request_escaped(method, url, body=None, headers=None): - validate_headers(headers) - url = encode_utf8(url) - if body: - body = encode_utf8(body) - func(method, url, body=body, headers=headers or {}) - return request_escaped - conn.request = request_wrapper(conn.request) - if proxy: - try: - # python 2.6 method - conn._set_tunnel(parsed.hostname, parsed.port) - except AttributeError: - # python 2.7 method - conn.set_tunnel(parsed.hostname, parsed.port) - return parsed, conn + :param data: Use data generator for chunked-transfer + :param files: Use files for default transfer + """ + return self.request('PUT', full_path, data, headers, files) + + def getresponse(self): + """ Adapt requests response to httplib interface """ + self.resp.status = self.resp.status_code + old_getheader = self.resp.raw.getheader + + def getheaders(): + return self.resp.headers.items() + + def getheader(k, v=None): + return old_getheader(k.lower(), v) + + self.resp.getheaders = getheaders + self.resp.getheader = getheader + self.resp.read = self.resp.raw.read + return self.resp + + +def http_connection(*arg, **kwarg): + """ :returns: tuple of (parsed url, connection object) """ + conn = HTTPConnection(*arg, **kwarg) + return conn.parsed_url, conn def get_auth_1_0(url, user, key, snet): @@ -893,27 +924,16 @@ def put_object(url, token=None, container=None, name=None, contents=None, if hasattr(contents, 'read'): if chunk_size is None: chunk_size = 65536 - conn.putrequest('PUT', path) - for header, value in headers.iteritems(): - conn.putheader(header, value) if content_length is None: - conn.putheader('Transfer-Encoding', 'chunked') - conn.endheaders() - chunk = contents.read(chunk_size) - while chunk: - conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - chunk = contents.read(chunk_size) - conn.send('0\r\n\r\n') + def chunk_reader(): + while True: + data = contents.read(chunk_size) + if not data: + break + yield data + conn.putrequest(path, headers=headers, data=chunk_reader()) else: - conn.endheaders() - left = content_length - while left > 0: - size = chunk_size - if size > left: - size = left - chunk = contents.read(size) - conn.send(chunk) - left -= len(chunk) + conn.putrequest(path, headers=headers, files={"file": contents}) else: if chunk_size is not None: warn_msg = '%s object has no \"read\" method, ignoring chunk_size'\ @@ -1132,6 +1152,8 @@ class Connection(object): def http_connection(self): return http_connection(self.url, + cacert=self.cacert, + insecure=self.insecure, ssl_compression=self.ssl_compression) def _add_response_dict(self, target_dict, kwargs): @@ -1163,7 +1185,9 @@ class Connection(object): rv = func(self.url, self.token, *args, **kwargs) self._add_response_dict(caller_response_dict, kwargs) return rv - except (socket.error, HTTPException) as e: + except SSLError: + raise + except (socket.error, RequestException) as e: self._add_response_dict(caller_response_dict, kwargs) if self.attempts > self.retries: logger.exception(e) diff --git a/swiftclient/https_connection.py b/swiftclient/https_connection.py deleted file mode 100644 index 2a2dc1f..0000000 --- a/swiftclient/https_connection.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2013 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -HTTPS/SSL related functionality -""" - -import socket - -from httplib import HTTPSConnection - -import OpenSSL - -try: - from eventlet.green.OpenSSL.SSL import GreenConnection - from eventlet.greenio import GreenSocket - from eventlet.patcher import is_monkey_patched - - def getsockopt(self, *args, **kwargs): - return self.fd.getsockopt(*args, **kwargs) - # The above is a workaround for an eventlet bug in getsockopt. - # TODO(mclaren): Workaround can be removed when this fix lands: - # https://bitbucket.org/eventlet/eventlet/commits/609f230 - GreenSocket.getsockopt = getsockopt -except ImportError: - def is_monkey_patched(*args): - return False - - -class HTTPSConnectionNoSSLComp(HTTPSConnection): - """ - Extended HTTPSConnection which uses the OpenSSL library - for disabling SSL compression. - Note: This functionality can eventually be replaced - with native Python 3.3 code. - """ - def __init__(self, host): - HTTPSConnection.__init__(self, host) - self.setcontext() - - def setcontext(self): - """ - Set up the OpenSSL context. - """ - self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) - # Disable SSL layer compression. - self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION - - def connect(self): - """ - Connect to an SSL port using the OpenSSL library and apply - per-connection parameters. - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock = OpenSSLConnectionDelegator(self.context, sock) - self.sock.connect((self.host, self.port)) - - -class OpenSSLConnectionDelegator(object): - """ - An OpenSSL.SSL.Connection delegator. - - Supplies an additional 'makefile' method which httplib requires - and is not present in OpenSSL.SSL.Connection. - - Note: Since it is not possible to inherit from OpenSSL.SSL.Connection - a delegator must be used. - """ - def __init__(self, *args, **kwargs): - if is_monkey_patched('socket'): - # If we are running in a monkey patched environment - # use eventlet's GreenConnection -- it handles eventlet's - # non-blocking sockets correctly. - Connection = GreenConnection - else: - Connection = OpenSSL.SSL.Connection - self.connection = Connection(*args, **kwargs) - - def __getattr__(self, name): - return getattr(self.connection, name) - - def makefile(self, *args, **kwargs): - return socket._fileobject(self.connection, *args, **kwargs) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index a74eada..a038dcc 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -14,23 +14,6 @@ # limitations under the License. """Miscellaneous utility functions for use with Swift.""" -import sys -import os - -_ver = sys.version_info - -#: Python 2.x? -is_py2 = (_ver[0] == 2) - -#: Python 3.x? -is_py3 = (_ver[0] == 3) - -if is_py2: - from urllib import getproxies, proxy_bypass -elif is_py3: - from urllib.request import getproxies, proxy_bypass - - TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) @@ -72,36 +55,3 @@ def prt_bytes(bytes, human_flag): bytes = '%12s' % bytes return(bytes) - - -# get_environ_proxies function, borrowed from python Requests -# (www.python-requests.org) -def get_environ_proxies(netloc): - """Return a dict of environment proxies.""" - - get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) - - # First check whether no_proxy is defined. If it is, check that the URL - # we're getting isn't in the no_proxy list. - no_proxy = get_proxy('no_proxy') - - if no_proxy: - # We need to check whether we match here. We need to see if we match - # the end of the netloc, both with and without the port. - no_proxy = no_proxy.replace(' ', '').split(',') - - for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): - # The URL does match something in no_proxy, so we don't want - # to apply the proxies on this URL. - return {} - - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - if proxy_bypass(netloc): - return {} - - # If we get here, we either didn't have no_proxy set or we're not going - # anywhere that no_proxy applies to, and the system settings don't require - # bypassing the proxy for the current URL. - return getproxies() |