diff options
Diffstat (limited to 'swiftclient/client.py')
| -rw-r--r-- | swiftclient/client.py | 240 |
1 files changed, 147 insertions, 93 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py index 7db75f0..f071182 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -17,6 +17,7 @@ OpenStack Swift client library used internally """ import socket +import re import requests import logging import warnings @@ -38,6 +39,7 @@ from swiftclient.utils import ( # Default is 100, increase to 256 http_client._MAXHEADERS = 256 +VERSIONFUL_AUTH_PATH = re.compile(r'v[2-3](?:\.0)?$') AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) @@ -58,6 +60,19 @@ except ImportError: def createLock(self): self.lock = None +ksexceptions = ksclient_v2 = ksclient_v3 = None +try: + from keystoneclient import exceptions as ksexceptions + # prevent keystoneclient warning us that it has no log handlers + logging.getLogger('keystoneclient').addHandler(NullHandler()) + from keystoneclient.v2_0 import client as ksclient_v2 +except ImportError: + pass +try: + from keystoneclient.v3 import client as ksclient_v3 +except ImportError: + pass + # requests version 1.2.3 try to encode headers in ascii, preventing # utf-8 encoded header to be 'prepared' if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): @@ -149,7 +164,7 @@ def http_log(args, kwargs, resp, body): elif element in ('GET', 'POST', 'PUT'): string_parts.append(' -X %s' % element) else: - string_parts.append(' %s' % element) + string_parts.append(' %s' % parse_header_string(element)) if 'headers' in kwargs: headers = scrub_headers(kwargs['headers']) for element in headers: @@ -233,8 +248,8 @@ def encode_meta_headers(headers): value = encode_utf8(value) header = header.lower() - if (isinstance(header, six.string_types) - and header.startswith(USER_METADATA_TYPE)): + if (isinstance(header, six.string_types) and + header.startswith(USER_METADATA_TYPE)): header = encode_utf8(header) ret[header] = value @@ -271,6 +286,9 @@ class _ObjectBody(object): def __next__(self): return self.next() + def close(self): + self.resp.close() + class _RetryBody(_ObjectBody): """ @@ -302,7 +320,7 @@ class _RetryBody(_ObjectBody): self.obj = obj self.query_string = query_string self.response_dict = response_dict - self.headers = headers if headers is not None else {} + self.headers = dict(headers) if headers is not None else {} self.bytes_read = 0 def read(self, length=None): @@ -384,6 +402,7 @@ class HTTPConnection(object): self.request_session = requests.Session() # Don't use requests's default headers self.request_session.headers = None + self.resp = None if self.parsed_url.scheme not in ('http', 'https'): raise ClientException('Unsupported scheme "%s" in url "%s"' % (self.parsed_url.scheme, url)) @@ -453,11 +472,23 @@ class HTTPConnection(object): self.resp.status = self.resp.status_code old_getheader = self.resp.raw.getheader + def _decode_header(string): + if string is None or six.PY2: + return string + return string.encode('iso-8859-1').decode('utf-8') + + def _encode_header(string): + if string is None or six.PY2: + return string + return string.encode('utf-8').decode('iso-8859-1') + def getheaders(): - return self.resp.headers.items() + return [(_decode_header(k), _decode_header(v)) + for k, v in self.resp.headers.items()] def getheader(k, v=None): - return old_getheader(k.lower(), v) + return _decode_header(old_getheader( + _encode_header(k.lower()), _encode_header(v))) def releasing_read(*args, **kwargs): chunk = self.resp.raw.read(*args, **kwargs) @@ -476,6 +507,11 @@ class HTTPConnection(object): return self.resp + def close(self): + if self.resp: + self.resp.close() + self.request_session.close() + def http_connection(*arg, **kwarg): """:returns: tuple of (parsed url, connection object)""" @@ -497,6 +533,8 @@ def get_auth_1_0(url, user, key, snet, **kwargs): conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() + resp.close() + conn.close() http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') @@ -511,8 +549,9 @@ def get_auth_1_0(url, user, key, snet, **kwargs): netloc = parsed[1] parsed[1] = 'snet-' + netloc url = urlunparse(parsed) - return url, resp.getheader('x-storage-token', - resp.getheader('x-auth-token')) + + token = resp.getheader('x-storage-token', resp.getheader('x-auth-token')) + return url, token def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): @@ -522,25 +561,6 @@ def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): return get_auth_keystone(auth_url, user, key, os_options, **kwargs) -def _import_keystone_client(auth_version): - # the attempted imports are encapsulated in this function to allow - # mocking for tests - try: - if auth_version in AUTH_VERSIONS_V3: - from keystoneclient.v3 import client as ksclient - else: - from keystoneclient.v2_0 import client as ksclient - from keystoneclient import exceptions - # prevent keystoneclient warning us that it has no log handlers - logging.getLogger('keystoneclient').addHandler(NullHandler()) - return ksclient, exceptions - except ImportError: - raise ClientException(''' -Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth -version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment -variables to be set or overridden with -A, -U, or -K.''') - - def get_auth_keystone(auth_url, user, key, os_options, **kwargs): """ Authenticate against a keystone server. @@ -555,7 +575,10 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): # Add the version suffix in case of versionless Keystone endpoints. If # auth_version is also unset it is likely that it is v3 - if len(urlparse(auth_url).path) <= 1: + if not VERSIONFUL_AUTH_PATH.match( + urlparse(auth_url).path.rstrip('/').rsplit('/', 1)[-1]): + # Normalize auth_url to end in a slash because urljoin + auth_url = auth_url.rstrip('/') + '/' if auth_version and auth_version in AUTH_VERSIONS_V2: auth_url = urljoin(auth_url, "v2.0") else: @@ -565,8 +588,21 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): # Legacy default if not set if auth_version is None: - auth_version = 'v2.0' - ksclient, exceptions = _import_keystone_client(auth_version) + auth_version = '2' + + ksclient = None + if auth_version in AUTH_VERSIONS_V3: + if ksclient_v3 is not None: + ksclient = ksclient_v3 + else: + if ksclient_v2 is not None: + ksclient = ksclient_v2 + + if ksclient is None: + raise ClientException(''' +Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth +version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment +variables to be set or overridden with -A, -U, or -K.''') try: _ksclient = ksclient.Client( @@ -587,13 +623,13 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): cert=kwargs.get('cert'), key=kwargs.get('cert_key'), auth_url=auth_url, insecure=insecure, timeout=timeout) - except exceptions.Unauthorized: + except ksexceptions.Unauthorized: msg = 'Unauthorized. Check username, password and tenant name/id.' if auth_version in AUTH_VERSIONS_V3: msg = ('Unauthorized. Check username/id, password, ' 'tenant name/id and user/tenant domain name/id.') raise ClientException(msg) - except exceptions.AuthorizationFailure as err: + except ksexceptions.AuthorizationFailure as err: raise ClientException('Authorization Failure. %s' % err) service_type = os_options.get('service_type') or 'object-store' endpoint_type = os_options.get('endpoint_type') or 'publicURL' @@ -606,7 +642,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): service_type=service_type, endpoint_type=endpoint_type, **filter_kwargs) - except exceptions.EndpointNotFound: + except ksexceptions.EndpointNotFound: raise ClientException('Endpoint for %s not found - ' 'have you specified a region?' % service_type) return endpoint, _ksclient.auth_token @@ -644,8 +680,10 @@ def get_auth(auth_url, user, key, **kwargs): if session: service_type = os_options.get('service_type', 'object-store') interface = os_options.get('endpoint_type', 'public') + region_name = os_options.get('region_name') storage_url = session.get_endpoint(service_type=service_type, - interface=interface) + interface=interface, + region_name=region_name) token = session.get_token() elif auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, @@ -668,9 +706,9 @@ def get_auth(auth_url, user, key, **kwargs): if kwargs.get('tenant_name'): os_options['tenant_name'] = kwargs['tenant_name'] - if not (os_options.get('tenant_name') or os_options.get('tenant_id') - or os_options.get('project_name') - or os_options.get('project_id')): + if not (os_options.get('tenant_name') or os_options.get('tenant_id') or + os_options.get('project_name') or + os_options.get('project_id')): if auth_version in AUTH_VERSIONS_V2: raise ClientException('No tenant specified') raise ClientException('No project name or project id specified.') @@ -719,7 +757,7 @@ def store_response(resp, response_dict): def get_account(url, token, marker=None, limit=None, prefix=None, end_marker=None, http_conn=None, full_listing=False, - service_token=None, headers=None): + service_token=None, headers=None, delimiter=None): """ Get a listing of containers for the account. @@ -735,6 +773,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, of 10000 listings :param service_token: service auth token :param headers: additional headers to include in the request + :param delimiter: delimiter query :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -748,14 +787,14 @@ def get_account(url, token, marker=None, limit=None, prefix=None, if not http_conn: http_conn = http_connection(url) if full_listing: - rv = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, headers=req_headers) + rv = get_account(url, token, marker, limit, prefix, end_marker, + http_conn, headers=req_headers, delimiter=delimiter) listing = rv[1] while listing: marker = listing[-1]['name'] listing = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, - headers=req_headers)[1] + end_marker, http_conn, headers=req_headers, + delimiter=delimiter)[1] if listing: rv[1].extend(listing) return rv @@ -767,6 +806,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) if end_marker: qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) @@ -847,13 +888,15 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, path = parsed.path if query_string: path += '?' + query_string - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request(method, path, data, headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request(method, path, data, req_headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {'headers': headers}, resp, body) + http_log((url, method,), {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -895,12 +938,6 @@ def get_container(url, token, container, marker=None, limit=None, """ if not http_conn: http_conn = http_connection(url) - if headers: - headers = dict(headers) - else: - headers = {} - headers['X-Auth-Token'] = token - headers['Accept-Encoding'] = 'gzip' if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, @@ -935,17 +972,20 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&path=%s' % quote(path) if query_string: qs += '&%s' % query_string.lstrip('?') + req_headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) method = 'GET' - conn.request(method, '%s?%s' % (cont_path, qs), '', headers) + conn.request(method, '%s?%s' % (cont_path, qs), '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%(url)s%(cont_path)s?%(qs)s' % {'url': url.replace(parsed.path, ''), 'cont_path': cont_path, 'qs': qs}, method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container GET failed', body) @@ -1018,23 +1058,23 @@ def put_container(url, token, container, headers=None, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'PUT' - if not headers: - headers = {} - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + if 'content-length' not in (k.lower() for k in req_headers): + req_headers['Content-Length'] = '0' if query_string: path += '?' + query_string.lstrip('?') - conn.request(method, path, '', headers) + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() store_response(resp, response_dict) http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container PUT failed', body) @@ -1061,16 +1101,18 @@ def post_container(url, token, container, headers, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'POST' - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' - conn.request(method, path, '', headers) + req_headers['Content-Length'] = '0' + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -1188,7 +1230,7 @@ def get_object(url, token, container, name, http_conn=None, def head_object(url, token, container, name, http_conn=None, - service_token=None, headers=None): + service_token=None, headers=None, query_string=None): """ Get object info @@ -1209,6 +1251,8 @@ def head_object(url, token, container, name, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + if query_string: + path += '?' + query_string if headers: headers = dict(headers) else: @@ -1365,14 +1409,16 @@ def post_object(url, token, container, name, headers, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request('POST', path, '', headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request('POST', path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'POST',), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -1540,7 +1586,7 @@ class Connection(object): os_options=None, auth_version="1", cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, retry_on_ratelimit=False, - timeout=None, session=None): + timeout=None, session=None, force_auth_retry=False): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1576,6 +1622,8 @@ class Connection(object): after a backoff. :param timeout: The connect timeout for the HTTP connection. :param session: A keystoneauth session object. + :param force_auth_retry: reset auth info even if client got unexpected + error except 401 Unauthorized. """ self.session = session self.authurl = authurl @@ -1608,16 +1656,14 @@ class Connection(object): self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit self.timeout = timeout + self.force_auth_retry = force_auth_retry def close(self): - if (self.http_conn and isinstance(self.http_conn, tuple) - and len(self.http_conn) > 1): + if (self.http_conn and isinstance(self.http_conn, tuple) and + len(self.http_conn) > 1): conn = self.http_conn[1] - if hasattr(conn, 'close') and callable(conn.close): - # XXX: Our HTTPConnection object has no close, should be - # trying to close the requests.Session here? - conn.close() - self.http_conn = None + conn.close() + self.http_conn = None def get_auth(self): self.url, self.token = get_auth(self.authurl, self.user, self.key, @@ -1677,10 +1723,10 @@ class Connection(object): try: if not self.url or not self.token: self.url, self.token = self.get_auth() - self.http_conn = None + self.close() if self.service_auth and not self.service_token: self.url, self.service_token = self.get_service_auth() - self.http_conn = None + self.close() self.auth_end_time = time() if not self.http_conn: self.http_conn = self.http_connection() @@ -1722,6 +1768,10 @@ class Connection(object): pass else: raise + + if self.force_auth_retry: + self.url = self.token = self.service_token = None + sleep(backoff) backoff = min(backoff * 2, self.max_backoff) if reset_func: @@ -1732,14 +1782,16 @@ class Connection(object): return self._retry(None, head_account, headers=headers) def get_account(self, marker=None, limit=None, prefix=None, - end_marker=None, full_listing=False, headers=None): + end_marker=None, full_listing=False, headers=None, + delimiter=None): """Wrapper for :func:`get_account`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just # retries where it left off. return self._retry(None, get_account, marker=marker, limit=limit, prefix=prefix, end_marker=end_marker, - full_listing=full_listing, headers=headers) + full_listing=full_listing, headers=headers, + delimiter=delimiter) def post_account(self, headers, response_dict=None, query_string=None, data=None): @@ -1785,9 +1837,10 @@ class Connection(object): query_string=query_string, headers=headers) - def head_object(self, container, obj, headers=None): + def head_object(self, container, obj, headers=None, query_string=None): """Wrapper for :func:`head_object`""" - return self._retry(None, head_object, container, obj, headers=headers) + return self._retry(None, head_object, container, obj, headers=headers, + query_string=query_string) def get_object(self, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): @@ -1832,7 +1885,9 @@ class Connection(object): reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() - reset_func = lambda *a, **k: seek(orig_pos) + + def reset_func(*a, **kw): + seek(orig_pos) elif reset: reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, @@ -1865,8 +1920,7 @@ class Connection(object): url = url or self.url if not url: url, _ = self.get_auth() - scheme = urlparse(url).scheme - netloc = urlparse(url).netloc - url = scheme + '://' + netloc + '/info' - http_conn = self.http_connection(url) - return get_capabilities(http_conn) + parsed = urlparse(urljoin(url, '/info')) + if not self.http_conn: + self.http_conn = self.http_connection(url) + return get_capabilities((parsed, self.http_conn[1])) |