Merge "crypto - purge crypto sysmeta from responses" into feature/crypto

5 files changed, 246 insertions, 23 deletions

diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py
index c5b7a1f59..7da98aaca 100644
--- a/swift/common/middleware/copy.py
+++ b/swift/common/middleware/copy.py
@@ -142,7 +142,7 @@ from swift.common.utils import get_logger, \
 from swift.common.swob import Request, HTTPPreconditionFailed, \
     HTTPRequestEntityTooLarge, HTTPBadRequest
 from swift.common.http import HTTP_MULTIPLE_CHOICES, HTTP_CREATED, \
-    is_success
+    is_success, HTTP_OK
 from swift.common.constraints import check_account_format, MAX_FILE_SIZE
 from swift.common.request_helpers import copy_header_subset, remove_items, \
     is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta
@@ -478,7 +478,26 @@ class ServerSideCopyMiddleware(object):
         # Set data source, content length and etag for the PUT request
         sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
         sink_req.content_length = source_resp.content_length
-        sink_req.etag = source_resp.etag
+        if (source_resp.status_int == HTTP_OK and
+                'X-Static-Large-Object' not in source_resp.headers and
+                ('X-Object-Manifest' not in source_resp.headers or
+                 req.params.get('multipart-manifest') == 'get')):
+            # copy source etag so that copied content is verified, unless:
+            #  - not a 200 OK response: source etag may not match the actual
+            #    content, for example with a 206 Partial Content response to a
+            #    ranged request
+            #  - SLO manifest: etag cannot be specified in manifest PUT; SLO
+            #    generates its own etag value which may differ from source
+            #  - SLO: etag in SLO response is not hash of actual content
+            #  - DLO: etag in DLO response is not hash of actual content
+            sink_req.headers['Etag'] = source_resp.etag
+        else:
+            # since we're not copying the source etag, make sure that any
+            # container update override values are not copied.
+            source_resp.headers.pop(
+                'X-Object-Sysmeta-Container-Update-Override-Etag', None)
+            source_resp.headers.pop(
+                'X-Object-Sysmeta-Container-Update-Override-Size', None)
 
         # We no longer need these headers
         sink_req.headers.pop('X-Copy-From', None)
diff --git a/swift/common/middleware/decrypter.py b/swift/common/middleware/decrypter.py
index fc3837017..c0cb6f099 100644
--- a/swift/common/middleware/decrypter.py
+++ b/swift/common/middleware/decrypter.py
@@ -25,7 +25,7 @@ from swift.common.middleware.crypto_utils import CryptoWSGIContext, \
     load_crypto_meta, extract_crypto_meta, Crypto
 from swift.common.exceptions import EncryptionException
 from swift.common.request_helpers import strip_user_meta_prefix, is_user_meta,\
-    get_object_transient_sysmeta, get_listing_content_type
+    get_object_transient_sysmeta, get_listing_content_type, get_sys_meta_prefix
 from swift.common.swob import Request, HTTPException, HTTPInternalServerError
 from swift.common.utils import get_logger, config_true_value, \
     parse_content_range, closing_if_possible, parse_content_type, \
@@ -34,6 +34,13 @@ from swift.common.utils import get_logger, config_true_value, \
 DECRYPT_CHUNK_SIZE = 65536
 
 
+def purge_crypto_sysmeta_headers(headers):
+    return [h for h in headers if not
+            h[0].lower().startswith(
+                (get_object_transient_sysmeta('crypto-'),
+                 get_sys_meta_prefix('object') + 'crypto-'))]
+
+
 class BaseDecrypterContext(CryptoWSGIContext):
     def __init__(self, decrypter, server_type, logger):
         super(BaseDecrypterContext, self).__init__(
@@ -313,6 +320,7 @@ class DecrypterObjContext(BaseDecrypterContext):
             # don't decrypt body of non-2xx responses
             resp_iter = app_resp
 
+        mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers)
         start_response(self._response_status, mod_resp_headers,
                        self._response_exc_info)
 
@@ -329,6 +337,7 @@ class DecrypterObjContext(BaseDecrypterContext):
                            self._response_exc_info)
         else:
             mod_resp_headers = self.decrypt_resp_headers(keys)
+            mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers)
             start_response(self._response_status, mod_resp_headers,
                            self._response_exc_info)
 
diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py
index 254203e63..db4e75050 100644
--- a/test/unit/common/middleware/test_copy.py
+++ b/test/unit/common/middleware/test_copy.py
@@ -602,6 +602,71 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
         self.assertEqual('PUT', self.authorized[1].method)
         self.assertEqual('/v1/a/c/o-copy', self.authorized[1].path)
 
+    def test_COPY_source_metadata(self):
+        source_headers = {
+            'x-object-sysmeta-test1': 'copy me',
+            'x-object-meta-test2': 'copy me too',
+            'x-object-transient-sysmeta-test3': 'ditto',
+            'x-object-sysmeta-container-update-override-etag': 'etag val',
+            'x-object-sysmeta-container-update-override-size': 'size val'}
+
+        self.app.register(
+            'GET', '/v1/a/c/o', swob.HTTPOk,
+            headers=source_headers.copy(), body='passed')
+
+        def verify_response(
+                expected_status, expected_headers, unexpected_headers):
+            self.assertEqual(expected_status, status)
+            for k, v in headers:
+                if k.lower() in expected_headers:
+                    expected_val = expected_headers.pop(k.lower())
+                    self.assertEqual(expected_val, v)
+                self.assertNotIn(k.lower(), unexpected_headers)
+            self.assertFalse(expected_headers)
+
+        # use a COPY request
+        self.app.register('PUT', '/v1/a/c/o-copy1', swob.HTTPCreated, {})
+        req = Request.blank('/v1/a/c/o', method='COPY',
+                            headers={'Content-Length': 0,
+                                     'Destination': 'c/o-copy1'})
+        status, headers, body = self.call_ssc(req)
+        verify_response('201 Created', source_headers.copy(), [])
+
+        req = Request.blank('/v1/a/c/o-copy1', method='GET')
+        status, headers, body = self.call_ssc(req)
+        verify_response('200 OK', source_headers.copy(), [])
+
+        # use a COPY request with a Range header
+        self.app.register('PUT', '/v1/a/c/o-copy1', swob.HTTPCreated, {})
+        req = Request.blank('/v1/a/c/o', method='COPY',
+                            headers={'Content-Length': 0,
+                                     'Destination': 'c/o-copy1',
+                                     'Range': 'bytes=1-2'})
+        status, headers, body = self.call_ssc(req)
+        expected_headers = source_headers.copy()
+        unexpected_headers = (
+            'x-object-sysmeta-container-update-override-etag',
+            'x-object-sysmeta-container-update-override-size')
+        for h in unexpected_headers:
+            expected_headers.pop(h)
+        verify_response('201 Created', expected_headers, unexpected_headers)
+
+        req = Request.blank('/v1/a/c/o-copy1', method='GET')
+        status, headers, body = self.call_ssc(req)
+        verify_response('200 OK', expected_headers, unexpected_headers)
+
+        # use a PUT with x-copy-from
+        self.app.register('PUT', '/v1/a/c/o-copy2', swob.HTTPCreated, {})
+        req = Request.blank('/v1/a/c/o-copy2', method='PUT',
+                            headers={'Content-Length': 0,
+                                     'X-Copy-From': 'c/o'})
+        status, headers, body = self.call_ssc(req)
+        verify_response('201 Created', source_headers.copy(), [])
+
+        req = Request.blank('/v1/a/c/o-copy2', method='GET')
+        status, headers, body = self.call_ssc(req)
+        verify_response('200 OK', source_headers.copy(), [])
+
     def test_COPY_no_destination_header(self):
         req = Request.blank(
             '/v1/a/c/o', method='COPY', headers={'Content-Length': 0})
diff --git a/test/unit/common/middleware/test_decrypter.py b/test/unit/common/middleware/test_decrypter.py
index e23303ad5..f405c2f1d 100644
--- a/test/unit/common/middleware/test_decrypter.py
+++ b/test/unit/common/middleware/test_decrypter.py
@@ -1125,6 +1125,27 @@ class TestModuleMethods(unittest.TestCase):
         self.assertTrue(callable(factory))
         self.assertIsInstance(factory(None), decrypter.Decrypter)
 
+    def test_purge_crypto_sysmeta_headers(self):
+        retained_headers = {'x-object-sysmeta-test1': 'keep',
+                            'x-object-meta-test2': 'retain',
+                            'x-object-transient-sysmeta-test3': 'leave intact',
+                            'etag': 'hold onto',
+                            'x-other': 'cherish',
+                            'x-object-not-meta': 'do not remove'}
+        purged_headers = {'x-object-sysmeta-crypto-test1': 'remove',
+                          'x-object-transient-sysmeta-crypto-test2': 'purge'}
+        test_headers = retained_headers.copy()
+        test_headers.update(purged_headers)
+        actual = decrypter.purge_crypto_sysmeta_headers(test_headers.items())
+
+        for k, v in actual:
+            k = k.lower()
+            self.assertNotIn(k, purged_headers)
+            if k in retained_headers:
+                self.assertEqual(retained_headers[k], v)
+                retained_headers.pop(k)
+        self.assertFalse(retained_headers)
+
 
 class TestDecrypter(unittest.TestCase):
     def test_app_exception(self):
diff --git a/test/unit/common/middleware/test_encrypter_decrypter.py b/test/unit/common/middleware/test_encrypter_decrypter.py
index 701c995b3..9f7e0a6f2 100644
--- a/test/unit/common/middleware/test_encrypter_decrypter.py
+++ b/test/unit/common/middleware/test_encrypter_decrypter.py
@@ -18,7 +18,7 @@ import unittest
 import uuid
 
 from swift.common import storage_policy
-from swift.common.middleware import encrypter, decrypter, keymaster
+from swift.common.middleware import encrypter, decrypter, keymaster, copy
 from swift.common.middleware.crypto_utils import load_crypto_meta, Crypto
 from swift.common.ring import Ring
 from swift.common.swob import Request
@@ -48,10 +48,6 @@ class TestCryptoPipelineChanges(unittest.TestCase):
             cls._test_context = None
 
     def setUp(self):
-        self.container_name = uuid.uuid4().hex
-        self.container_path = 'http://localhost:8080/v1/a/' + \
-                              self.container_name
-        self.object_path = self.container_path + '/o'
         self.plaintext = 'unencrypted body content'
         self.plaintext_etag = md5hex(self.plaintext)
 
@@ -63,15 +59,21 @@ class TestCryptoPipelineChanges(unittest.TestCase):
         self.km = keymaster.KeyMaster(enc, TEST_KEYMASTER_CONF)
         self.crypto_app = decrypter.Decrypter(self.km, {})
 
-    def _create_container(self, app, policy_name='one'):
+    def _create_container(self, app, policy_name='one', container_path=None):
+        if not container_path:
+            # choose new container name so that the policy can be specified
+            self.container_name = uuid.uuid4().hex
+            self.container_path = 'http://foo:8080/v1/a/' + self.container_name
+            self.object_path = self.container_path + '/o'
+            container_path = self.container_path
         req = Request.blank(
-            self.container_path, method='PUT',
+            container_path, method='PUT',
             headers={'X-Storage-Policy': policy_name})
         resp = req.get_response(app)
         self.assertEqual('201 Created', resp.status)
         # sanity check
         req = Request.blank(
-            self.container_path, method='HEAD',
+            container_path, method='HEAD',
             headers={'X-Storage-Policy': policy_name})
         resp = req.get_response(app)
         self.assertEqual(policy_name, resp.headers['X-Storage-Policy'])
@@ -92,25 +94,35 @@ class TestCryptoPipelineChanges(unittest.TestCase):
         self.assertEqual('202 Accepted', resp.status)
         return resp
 
-    def _check_GET_and_HEAD(self, app):
-        req = Request.blank(self.object_path, method='GET')
+    def _copy_object(self, app, destination):
+        req = Request.blank(self.object_path, method='COPY',
+                            headers={'Destination': destination})
+        resp = req.get_response(app)
+        self.assertEqual('201 Created', resp.status)
+        self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
+        return resp
+
+    def _check_GET_and_HEAD(self, app, object_path=None):
+        object_path = object_path or self.object_path
+        req = Request.blank(object_path, method='GET')
         resp = req.get_response(app)
         self.assertEqual('200 OK', resp.status)
         self.assertEqual(self.plaintext, resp.body)
         self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
 
-        req = Request.blank(self.object_path, method='HEAD')
+        req = Request.blank(object_path, method='HEAD')
         resp = req.get_response(app)
         self.assertEqual('200 OK', resp.status)
         self.assertEqual('', resp.body)
         self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
 
-    def _check_match_requests(self, method, app):
+    def _check_match_requests(self, method, app, object_path=None):
+        object_path = object_path or self.object_path
         # verify conditional match requests
         expected_body = self.plaintext if method == 'GET' else ''
 
         # If-Match matches
-        req = Request.blank(self.object_path, method=method,
+        req = Request.blank(object_path, method=method,
                             headers={'If-Match': '"%s"' % self.plaintext_etag})
         resp = req.get_response(app)
         self.assertEqual('200 OK', resp.status)
@@ -119,7 +131,7 @@ class TestCryptoPipelineChanges(unittest.TestCase):
         self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
 
         # If-Match wildcard
-        req = Request.blank(self.object_path, method=method,
+        req = Request.blank(object_path, method=method,
                             headers={'If-Match': '*'})
         resp = req.get_response(app)
         self.assertEqual('200 OK', resp.status)
@@ -128,7 +140,7 @@ class TestCryptoPipelineChanges(unittest.TestCase):
         self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
 
         # If-Match does not match
-        req = Request.blank(self.object_path, method=method,
+        req = Request.blank(object_path, method=method,
                             headers={'If-Match': '"not the etag"'})
         resp = req.get_response(app)
         self.assertEqual('412 Precondition Failed', resp.status)
@@ -137,7 +149,7 @@ class TestCryptoPipelineChanges(unittest.TestCase):
 
         # If-None-Match matches
         req = Request.blank(
-            self.object_path, method=method,
+            object_path, method=method,
             headers={'If-None-Match': '"%s"' % self.plaintext_etag})
         resp = req.get_response(app)
         self.assertEqual('304 Not Modified', resp.status)
@@ -145,7 +157,7 @@ class TestCryptoPipelineChanges(unittest.TestCase):
         self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
 
         # If-None-Match wildcard
-        req = Request.blank(self.object_path, method=method,
+        req = Request.blank(object_path, method=method,
                             headers={'If-None-Match': '*'})
         resp = req.get_response(app)
         self.assertEqual('304 Not Modified', resp.status)
@@ -153,7 +165,7 @@ class TestCryptoPipelineChanges(unittest.TestCase):
         self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
 
         # If-None-Match does not match
-        req = Request.blank(self.object_path, method=method,
+        req = Request.blank(object_path, method=method,
                             headers={'If-None-Match': '"not the etag"'})
         resp = req.get_response(app)
         self.assertEqual('200 OK', resp.status)
@@ -161,9 +173,10 @@ class TestCryptoPipelineChanges(unittest.TestCase):
         self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
         self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
 
-    def _check_listing(self, app, expect_mismatch=False):
+    def _check_listing(self, app, expect_mismatch=False, container_path=None):
+        container_path = container_path or self.container_path
         req = Request.blank(
-            self.container_path, method='GET', query_string='format=json')
+            container_path, method='GET', query_string='format=json')
         resp = req.get_response(app)
         self.assertEqual('200 OK', resp.status)
         listing = json.loads(resp.body)
@@ -447,6 +460,102 @@ class TestCryptoPipelineChanges(unittest.TestCase):
             frags = [frag for node, frag in frag_selection]
             self.assertEqual(exp_body, policy.pyeclib_driver.decode(frags))
 
+    def _test_copy_encrypted_to_encrypted(
+            self, src_policy_name, dest_policy_name):
+        self._create_container(self.proxy_app, policy_name=src_policy_name)
+        self._put_object(self.crypto_app, self.plaintext)
+        self._post_object(self.crypto_app)
+
+        copy_crypto_app = copy.ServerSideCopyMiddleware(self.crypto_app, {})
+
+        dest_container = uuid.uuid4().hex
+        dest_container_path = 'http://localhost:8080/v1/a/' + dest_container
+        self._create_container(copy_crypto_app, policy_name=dest_policy_name,
+                               container_path=dest_container_path)
+        dest_obj_path = dest_container_path + '/o'
+        dest = '/%s/%s' % (dest_container, 'o')
+        self._copy_object(copy_crypto_app, dest)
+
+        self._check_GET_and_HEAD(copy_crypto_app, object_path=dest_obj_path)
+        self._check_listing(
+            copy_crypto_app, container_path=dest_container_path)
+        self._check_match_requests(
+            'GET', copy_crypto_app, object_path=dest_obj_path)
+        self._check_match_requests(
+            'HEAD', copy_crypto_app, object_path=dest_obj_path)
+
+    def test_copy_encrypted_to_encrypted(self):
+        self._test_copy_encrypted_to_encrypted('ec', 'ec')
+        self._test_copy_encrypted_to_encrypted('one', 'ec')
+        self._test_copy_encrypted_to_encrypted('ec', 'one')
+        self._test_copy_encrypted_to_encrypted('one', 'one')
+
+    def _test_copy_encrypted_to_unencrypted(
+            self, src_policy_name, dest_policy_name):
+        self._create_container(self.proxy_app, policy_name=src_policy_name)
+        self._put_object(self.crypto_app, self.plaintext)
+        self._post_object(self.crypto_app)
+
+        # make a pipeline with encryption disabled, use it to copy object
+        enc = encrypter.Encrypter(
+            self.proxy_app, {'disable_encryption': 'true'})
+        km = keymaster.KeyMaster(enc, TEST_KEYMASTER_CONF)
+        dec = decrypter.Decrypter(km, {})
+        copy_app = copy.ServerSideCopyMiddleware(dec, {})
+
+        dest_container = uuid.uuid4().hex
+        dest_container_path = 'http://localhost:8080/v1/a/' + dest_container
+        self._create_container(self.crypto_app, policy_name=dest_policy_name,
+                               container_path=dest_container_path)
+        dest_obj_path = dest_container_path + '/o'
+        dest = '/%s/%s' % (dest_container, 'o')
+        self._copy_object(copy_app, dest)
+
+        self._check_GET_and_HEAD(copy_app, object_path=dest_obj_path)
+        self._check_GET_and_HEAD(self.proxy_app, object_path=dest_obj_path)
+        self._check_listing(copy_app, container_path=dest_container_path)
+        self._check_listing(self.proxy_app, container_path=dest_container_path)
+        self._check_match_requests(
+            'GET', self.proxy_app, object_path=dest_obj_path)
+        self._check_match_requests(
+            'HEAD', self.proxy_app, object_path=dest_obj_path)
+
+    def test_copy_encrypted_to_unencrypted(self):
+        self._test_copy_encrypted_to_unencrypted('ec', 'ec')
+        self._test_copy_encrypted_to_unencrypted('one', 'ec')
+        self._test_copy_encrypted_to_unencrypted('ec', 'one')
+        self._test_copy_encrypted_to_unencrypted('one', 'one')
+
+    def _test_copy_unencrypted_to_encrypted(
+            self, src_policy_name, dest_policy_name):
+        self._create_container(self.proxy_app, policy_name=src_policy_name)
+        self._put_object(self.proxy_app, self.plaintext)
+        self._post_object(self.proxy_app)
+
+        copy_crypto_app = copy.ServerSideCopyMiddleware(self.crypto_app, {})
+
+        dest_container = uuid.uuid4().hex
+        dest_container_path = 'http://localhost:8080/v1/a/' + dest_container
+        self._create_container(copy_crypto_app, policy_name=dest_policy_name,
+                               container_path=dest_container_path)
+        dest_obj_path = dest_container_path + '/o'
+        dest = '/%s/%s' % (dest_container, 'o')
+        self._copy_object(copy_crypto_app, dest)
+
+        self._check_GET_and_HEAD(copy_crypto_app, object_path=dest_obj_path)
+        self._check_listing(
+            copy_crypto_app, container_path=dest_container_path)
+        self._check_match_requests(
+            'GET', copy_crypto_app, object_path=dest_obj_path)
+        self._check_match_requests(
+            'HEAD', copy_crypto_app, object_path=dest_obj_path)
+
+    def test_copy_unencrypted_to_encrypted(self):
+        self._test_copy_unencrypted_to_encrypted('ec', 'ec')
+        self._test_copy_unencrypted_to_encrypted('one', 'ec')
+        self._test_copy_unencrypted_to_encrypted('ec', 'one')
+        self._test_copy_unencrypted_to_encrypted('one', 'one')
+
 
 class TestCryptoPipelineChangesFastPost(TestCryptoPipelineChanges):
     @classmethod