diff options
Diffstat (limited to 'swiftclient/client.py')
| -rw-r--r-- | swiftclient/client.py | 196 |
1 files changed, 112 insertions, 84 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py index 6990ea3..807cb3d 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -72,6 +72,69 @@ if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): logger = logging.getLogger("swiftclient") logger.addHandler(NullHandler()) +#: Default behaviour is to redact header values known to contain secrets, +#: such as ``X-Auth-Key`` and ``X-Auth-Token``. Up to the first 16 chars +#: may be revealed. +#: +#: To disable, set the value of ``redact_sensitive_headers`` to ``False``. +#: +#: When header redaction is enabled, ``reveal_sensitive_prefix`` configures the +#: maximum length of any sensitive header data sent to the logs. If the header +#: is less than twice this length, only ``int(len(value)/2)`` chars will be +#: logged; if it is less than 15 chars long, even less will be logged. +logger_settings = { + 'redact_sensitive_headers': True, + 'reveal_sensitive_prefix': 16 +} +#: A list of sensitive headers to redact in logs. Note that when extending this +#: list, the header names must be added in all lower case. +LOGGER_SENSITIVE_HEADERS = [ + 'x-auth-token', 'x-auth-key', 'x-service-token', 'x-storage-token', + 'x-account-meta-temp-url-key', 'x-account-meta-temp-url-key-2', + 'x-container-meta-temp-url-key', 'x-container-meta-temp-url-key-2', + 'set-cookie' +] + + +def safe_value(name, value): + """ + Only show up to logger_settings['reveal_sensitive_prefix'] characters + from a sensitive header. + + :param name: Header name + :param value: Header value + :return: Safe header value + """ + if name.lower() in LOGGER_SENSITIVE_HEADERS: + prefix_length = logger_settings.get('reveal_sensitive_prefix', 16) + prefix_length = int( + min(prefix_length, (len(value) ** 2) / 32, len(value) / 2) + ) + redacted_value = value[0:prefix_length] + return redacted_value + '...' + return value + + +def scrub_headers(headers): + """ + Redact header values that can contain sensitive information that + should not be logged. + + :param headers: Either a dict or an iterable of two-element tuples + :return: Safe dictionary of headers with sensitive information removed + """ + if isinstance(headers, dict): + headers = headers.items() + headers = [ + (parse_header_string(key), parse_header_string(val)) + for (key, val) in headers + ] + if not logger_settings.get('redact_sensitive_headers', True): + return dict(headers) + if logger_settings.get('reveal_sensitive_prefix', 16) < 0: + logger_settings['reveal_sensitive_prefix'] = 16 + return {key: safe_value(key, val) for (key, val) in headers} + def http_log(args, kwargs, resp, body): if not logger.isEnabledFor(logging.INFO): @@ -87,8 +150,9 @@ def http_log(args, kwargs, resp, body): else: string_parts.append(' %s' % element) if 'headers' in kwargs: - for element in kwargs['headers']: - header = ' -H "%s: %s"' % (element, kwargs['headers'][element]) + headers = scrub_headers(kwargs['headers']) + for element in headers: + header = ' -H "%s: %s"' % (element, headers[element]) string_parts.append(header) # log response as debug if good, or info if error @@ -99,12 +163,14 @@ def http_log(args, kwargs, resp, body): log_method("REQ: %s", "".join(string_parts)) log_method("RESP STATUS: %s %s", resp.status, resp.reason) - log_method("RESP HEADERS: %s", resp.getheaders()) + log_method("RESP HEADERS: %s", scrub_headers(resp.getheaders())) if body: log_method("RESP BODY: %s", body) def parse_header_string(data): + if not isinstance(data, (six.text_type, six.binary_type)): + data = str(data) if six.PY2: if isinstance(data, six.text_type): # Under Python2 requests only returns binary_type, but if we get @@ -403,20 +469,18 @@ def get_auth_1_0(url, user, key, snet, **kwargs): parsed, conn = http_connection(url, cacert=cacert, insecure=insecure, timeout=timeout) method = 'GET' - conn.request(method, parsed.path, '', - {'X-Auth-User': user, 'X-Auth-Key': key}) + headers = {'X-Auth-User': user, 'X-Auth-Key': key} + conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {}, resp, body) + http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') # There is a side-effect on current Rackspace 1.0 server where a # bad URL would get you that document page and a 200. We error out # if we don't have a x-storage-url header and if we get a body. if resp.status < 200 or resp.status >= 300 or (body and not url): - raise ClientException('Auth GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason) + raise ClientException.from_response(resp, 'Auth GET failed', body) if snet: parsed = list(urlparse(url)) # Second item in the list is the netloc @@ -471,6 +535,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): _ksclient = ksclient.Client( username=user, password=key, + token=os_options.get('auth_token'), tenant_name=os_options.get('tenant_name'), tenant_id=os_options.get('tenant_id'), user_id=os_options.get('user_id'), @@ -655,11 +720,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, resp_headers = resp_header_dict(resp) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_query=qs, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account GET failed', body) if resp.status == 204: return resp_headers, [] return resp_headers, parse_api_response(resp_headers, body) @@ -691,16 +752,13 @@ def head_account(url, token, http_conn=None, service_token=None): body = resp.read() http_log((url, method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers def post_account(url, token, headers, http_conn=None, response_dict=None, - service_token=None): + service_token=None, query_string=None, data=None): """ Update an account's metadata. @@ -712,17 +770,23 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token + :param query_string: if set will be appended with '?' to generated path + :param data: an optional message body for the request :raises ClientException: HTTP POST request failed + :returns: resp_headers, body """ if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) method = 'POST' + path = parsed.path + if query_string: + path += '?' + query_string headers['X-Auth-Token'] = token if service_token: headers['X-Service-Token'] = service_token - conn.request(method, parsed.path, '', headers) + conn.request(method, path, data, headers) resp = conn.getresponse() body = resp.read() http_log((url, method,), {'headers': headers}, resp, body) @@ -730,13 +794,11 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account POST failed', - http_scheme=parsed.scheme, - http_host=conn.host, - http_path=parsed.path, - http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account POST failed', body) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, body def get_container(url, token, container, marker=None, limit=None, @@ -775,7 +837,7 @@ def get_container(url, token, container, marker=None, limit=None, if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, - service_token, headers=headers) + service_token=service_token, headers=headers) listing = rv[1] while listing: if not delimiter: @@ -784,7 +846,7 @@ def get_container(url, token, container, marker=None, limit=None, marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, - http_conn, service_token, + http_conn, service_token=service_token, headers=headers)[1] if listing: rv[1].extend(listing) @@ -817,11 +879,7 @@ def get_container(url, token, container, marker=None, limit=None, {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container GET failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=cont_path, http_query=qs, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Container GET failed', body) resp_headers = resp_header_dict(resp) if resp.status == 204: return resp_headers, [] @@ -862,11 +920,8 @@ def head_container(url, token, container, http_conn=None, headers=None, {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container HEAD failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -909,11 +964,7 @@ def put_container(url, token, container, headers=None, http_conn=None, http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container PUT failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Container PUT failed', body) def post_container(url, token, container, headers, http_conn=None, @@ -952,11 +1003,8 @@ def post_container(url, token, container, headers, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container POST failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container POST failed', body) def delete_container(url, token, container, http_conn=None, @@ -992,11 +1040,8 @@ def delete_container(url, token, container, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container DELETE failed', body) def get_object(url, token, container, name, http_conn=None, @@ -1049,11 +1094,7 @@ def get_object(url, token, container, name, http_conn=None, body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) - raise ClientException('Object GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object GET failed', body) if resp_chunk_size: object_body = _ObjectBody(resp, resp_chunk_size) else: @@ -1100,10 +1141,7 @@ def head_object(url, token, container, name, http_conn=None, http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -1209,17 +1247,13 @@ def put_object(url, token=None, container=None, name=None, contents=None, resp = conn.getresponse() body = resp.read() - headers = {'X-Auth-Token': token} http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'PUT',), {'headers': headers}, resp, body) store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object PUT failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object PUT failed', body) etag = resp.getheader('etag', '').strip('"') return etag @@ -1259,10 +1293,7 @@ def post_object(url, token, container, name, headers, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object POST failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object POST failed', body) def delete_object(url, token=None, container=None, name=None, http_conn=None, @@ -1316,11 +1347,7 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object DELETE failed', body) def get_capabilities(http_conn): @@ -1337,11 +1364,8 @@ def get_capabilities(http_conn): body = resp.read() http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Capabilities GET failed', - http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Capabilities GET failed', body) resp_headers = resp_header_dict(resp) return parse_api_response(resp_headers, body) @@ -1555,9 +1579,11 @@ class Connection(object): prefix=prefix, end_marker=end_marker, full_listing=full_listing) - def post_account(self, headers, response_dict=None): + def post_account(self, headers, response_dict=None, + query_string=None, data=None): """Wrapper for :func:`post_account`""" return self._retry(None, post_account, headers, + query_string=query_string, data=data, response_dict=response_dict) def head_container(self, container, headers=None): @@ -1635,10 +1661,12 @@ class Connection(object): if self.retries > 0: tell = getattr(contents, 'tell', None) seek = getattr(contents, 'seek', None) + reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() reset_func = lambda *a, **k: seek(orig_pos) - + elif reset: + reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, content_length=content_length, etag=etag, chunk_size=chunk_size, content_type=content_type, |