# Copyright (c) 2015-2016 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import base64 import hashlib import hmac from contextlib import contextmanager from swift.common.constraints import check_metadata from swift.common.http import is_success from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \ dump_crypto_meta, append_crypto_meta, Crypto from swift.common.request_helpers import get_object_transient_sysmeta, \ strip_user_meta_prefix, is_user_meta, update_etag_is_at_header from swift.common.swob import Request, Match, HTTPException, \ HTTPUnprocessableEntity from swift.common.utils import get_logger, config_true_value, \ MD5_OF_EMPTY_STRING def encrypt_header_val(crypto, value, key): """ Encrypt a header value using the supplied key. :param crypto: a Crypto instance :param value: value to encrypt :param key: crypto key to use :returns: a tuple of (encrypted value, crypto_meta) where crypto_meta is a dict of form returned by :py:func:`~swift.common.middleware.crypto.Crypto.get_crypto_meta` :raises ValueError: if value is empty """ if not value: raise ValueError('empty value is not acceptable') crypto_meta = crypto.create_crypto_meta() crypto_ctxt = crypto.create_encryption_ctxt(key, crypto_meta['iv']) enc_val = base64.b64encode(crypto_ctxt.update(value)) return enc_val, crypto_meta def _hmac_etag(key, etag): """ Compute an HMAC-SHA256 using given key and etag. :param key: The starting key for the hash. :param etag: The etag to hash. :returns: a Base64-encoded representation of the HMAC """ result = hmac.new(key, etag, digestmod=hashlib.sha256).digest() return base64.b64encode(result).decode() class EncInputWrapper(object): """File-like object to be swapped in for wsgi.input.""" def __init__(self, crypto, keys, req, logger): self.env = req.environ self.wsgi_input = req.environ['wsgi.input'] self.path = req.path self.crypto = crypto self.body_crypto_ctxt = None self.keys = keys self.plaintext_md5 = None self.ciphertext_md5 = None self.logger = logger self.install_footers_callback(req) def _init_encryption_context(self): # do this once when body is first read if self.body_crypto_ctxt is None: self.body_crypto_meta = self.crypto.create_crypto_meta() body_key = self.crypto.create_random_key() # wrap the body key with object key self.body_crypto_meta['body_key'] = self.crypto.wrap_key( self.keys['object'], body_key) self.body_crypto_meta['key_id'] = self.keys['id'] self.body_crypto_ctxt = self.crypto.create_encryption_ctxt( body_key, self.body_crypto_meta.get('iv')) self.plaintext_md5 = hashlib.md5() self.ciphertext_md5 = hashlib.md5() def install_footers_callback(self, req): # the proxy controller will call back for footer metadata after # body has been sent inner_callback = req.environ.get('swift.callback.update_footers') # remove any Etag from headers, it won't be valid for ciphertext and # we'll send the ciphertext Etag later in footer metadata client_etag = req.headers.pop('etag', None) container_listing_etag_header = req.headers.get( 'X-Object-Sysmeta-Container-Update-Override-Etag') def footers_callback(footers): if inner_callback: # pass on footers dict to any other callback that was # registered before this one. It may override any footers that # were set. inner_callback(footers) plaintext_etag = None if self.body_crypto_ctxt: plaintext_etag = self.plaintext_md5.hexdigest() # If client (or other middleware) supplied etag, then validate # against plaintext etag etag_to_check = footers.get('Etag') or client_etag if (etag_to_check is not None and plaintext_etag != etag_to_check): raise HTTPUnprocessableEntity(request=Request(self.env)) # override any previous notion of etag with the ciphertext etag footers['Etag'] = self.ciphertext_md5.hexdigest() # Encrypt the plaintext etag using the object key and persist # as sysmeta along with the crypto parameters that were used. encrypted_etag, etag_crypto_meta = encrypt_header_val( self.crypto, plaintext_etag, self.keys['object']) footers['X-Object-Sysmeta-Crypto-Etag'] = \ append_crypto_meta(encrypted_etag, etag_crypto_meta) footers['X-Object-Sysmeta-Crypto-Body-Meta'] = \ dump_crypto_meta(self.body_crypto_meta) # Also add an HMAC of the etag for use when evaluating # conditional requests footers['X-Object-Sysmeta-Crypto-Etag-Mac'] = _hmac_etag( self.keys['object'], plaintext_etag) else: # No data was read from body, nothing was encrypted, so don't # set any crypto sysmeta for the body, but do re-instate any # etag provided in inbound request if other middleware has not # already set a value. if client_etag is not None: footers.setdefault('Etag', client_etag) # When deciding on the etag that should appear in container # listings, look for: # * override in the footer, otherwise # * override in the header, and finally # * MD5 of the plaintext received # This may be None if no override was set and no data was read container_listing_etag = footers.get( 'X-Object-Sysmeta-Container-Update-Override-Etag', container_listing_etag_header) or plaintext_etag if (container_listing_etag is not None and (container_listing_etag != MD5_OF_EMPTY_STRING or plaintext_etag)): # Encrypt the container-listing etag using the container key # and a random IV, and use it to override the container update # value, with the crypto parameters appended. We use the # container key here so that only that key is required to # decrypt all etag values in a container listing when handling # a container GET request. Don't encrypt an EMPTY_ETAG # unless there actually was some body content, in which case # the container-listing etag is possibly conveying some # non-obvious information. val, crypto_meta = encrypt_header_val( self.crypto, container_listing_etag, self.keys['container']) crypto_meta['key_id'] = self.keys['id'] footers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \ append_crypto_meta(val, crypto_meta) # else: no override was set and no data was read req.environ['swift.callback.update_footers'] = footers_callback def read(self, *args, **kwargs): return self.readChunk(self.wsgi_input.read, *args, **kwargs) def readline(self, *args, **kwargs): return self.readChunk(self.wsgi_input.readline, *args, **kwargs) def readChunk(self, read_method, *args, **kwargs): chunk = read_method(*args, **kwargs) if chunk: self._init_encryption_context() self.plaintext_md5.update(chunk) # Encrypt one chunk at a time ciphertext = self.body_crypto_ctxt.update(chunk) self.ciphertext_md5.update(ciphertext) return ciphertext return chunk class EncrypterObjContext(CryptoWSGIContext): def __init__(self, encrypter, logger): super(EncrypterObjContext, self).__init__( encrypter, 'object', logger) def _check_headers(self, req): # Check the user-metadata length before encrypting and encoding error_response = check_metadata(req, self.server_type) if error_response: raise error_response def encrypt_user_metadata(self, req, keys): """ Encrypt user-metadata header values. Replace each x-object-meta- user metadata header with a corresponding x-object-transient-sysmeta-crypto-meta- header which has the crypto metadata required to decrypt appended to the encrypted value. :param req: a swob Request :param keys: a dict of encryption keys """ prefix = get_object_transient_sysmeta('crypto-meta-') user_meta_headers = [h for h in req.headers.items() if is_user_meta(self.server_type, h[0]) and h[1]] crypto_meta = None for name, val in user_meta_headers: short_name = strip_user_meta_prefix(self.server_type, name) new_name = prefix + short_name enc_val, crypto_meta = encrypt_header_val( self.crypto, val, keys[self.server_type]) req.headers[new_name] = append_crypto_meta(enc_val, crypto_meta) req.headers.pop(name) # store a single copy of the crypto meta items that are common to all # encrypted user metadata independently of any such meta that is stored # with the object body because it might change on a POST. This is done # for future-proofing - the meta stored here is not currently used # during decryption. if crypto_meta: meta = dump_crypto_meta({'cipher': crypto_meta['cipher'], 'key_id': keys['id']}) req.headers[get_object_transient_sysmeta('crypto-meta')] = meta def handle_put(self, req, start_response): self._check_headers(req) keys = self.get_keys(req.environ, required=['object', 'container']) self.encrypt_user_metadata(req, keys) enc_input_proxy = EncInputWrapper(self.crypto, keys, req, self.logger) req.environ['wsgi.input'] = enc_input_proxy resp = self._app_call(req.environ) # If an etag is in the response headers and a plaintext etag was # calculated, then overwrite the response value with the plaintext etag # provided it matches the ciphertext etag. If it does not match then do # not overwrite and allow the response value to return to client. mod_resp_headers = self._response_headers if (is_success(self._get_status_int()) and enc_input_proxy.plaintext_md5): plaintext_etag = enc_input_proxy.plaintext_md5.hexdigest() ciphertext_etag = enc_input_proxy.ciphertext_md5.hexdigest() mod_resp_headers = [ (h, v if (h.lower() != 'etag' or v.strip('"') != ciphertext_etag) else plaintext_etag) for h, v in mod_resp_headers] start_response(self._response_status, mod_resp_headers, self._response_exc_info) return resp def handle_post(self, req, start_response): """ Encrypt the new object headers with a new iv and the current crypto. Note that an object may have encrypted headers while the body may remain unencrypted. """ self._check_headers(req) keys = self.get_keys(req.environ) self.encrypt_user_metadata(req, keys) resp = self._app_call(req.environ) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp @contextmanager def _mask_conditional_etags(self, req, header_name): """ Calculate HMACs of etags in header value and append to existing list. The HMACs are calculated in the same way as was done for the object plaintext etag to generate the value of X-Object-Sysmeta-Crypto-Etag-Mac when the object was PUT. The object server can therefore use these HMACs to evaluate conditional requests. The existing etag values are left in the list of values to match in case the object was not encrypted when it was PUT. It is unlikely that a masked etag value would collide with an unmasked value. :param req: an instance of swob.Request :param header_name: name of header that has etags to mask :return: True if any etags were masked, False otherwise """ masked = False old_etags = req.headers.get(header_name) if old_etags: keys = self.get_keys(req.environ) new_etags = [] for etag in Match(old_etags).tags: if etag == '*': new_etags.append(etag) continue masked_etag = _hmac_etag(keys['object'], etag) new_etags.extend(('"%s"' % etag, '"%s"' % masked_etag)) masked = True req.headers[header_name] = ', '.join(new_etags) try: yield masked finally: if old_etags: req.headers[header_name] = old_etags def handle_get_or_head(self, req, start_response): with self._mask_conditional_etags(req, 'If-Match') as masked1: with self._mask_conditional_etags(req, 'If-None-Match') as masked2: if masked1 or masked2: update_etag_is_at_header( req, 'X-Object-Sysmeta-Crypto-Etag-Mac') resp = self._app_call(req.environ) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp class Encrypter(object): """Middleware for encrypting data and user metadata. By default all PUT or POST'ed object data and/or metadata will be encrypted. Encryption of new data and/or metadata may be disabled by setting the ``disable_encryption`` option to True. However, this middleware should remain in the pipeline in order for existing encrypted data to be read. """ def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route="encrypter") self.crypto = Crypto(conf) self.disable_encryption = config_true_value( conf.get('disable_encryption', 'false')) def __call__(self, env, start_response): # If override is set in env, then just pass along if config_true_value(env.get('swift.crypto.override')): return self.app(env, start_response) req = Request(env) if self.disable_encryption and req.method in ('PUT', 'POST'): return self.app(env, start_response) try: req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) if req.method in ('GET', 'HEAD'): handler = EncrypterObjContext(self, self.logger).handle_get_or_head elif req.method == 'PUT': handler = EncrypterObjContext(self, self.logger).handle_put elif req.method == 'POST': handler = EncrypterObjContext(self, self.logger).handle_post else: # anything else return self.app(env, start_response) try: return handler(req, start_response) except HTTPException as err_resp: return err_resp(env, start_response)