path: root/test/functional/test_dlo.py

#!/usr/bin/python -u
# Copyright (c) 2010-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.common.swob import str_to_wsgi
import test.functional as tf
from test.functional.tests import Utils, Base, Base2, BaseEnv
from test.functional.swift_test_client import Connection, ResponseError


def setUpModule():
    tf.setup_package()


def tearDownModule():
    tf.teardown_package()


class TestDloEnv(BaseEnv):
    @classmethod
    def setUp(cls):
        super(TestDloEnv, cls).setUp()

        cls.container = cls.account.container(Utils.create_name())
        cls.container2 = cls.account.container(Utils.create_name())

        for cont in (cls.container, cls.container2):
            if not cont.create():
                raise ResponseError(cls.conn.response)

        prefix = Utils.create_name(10)
        cls.segment_prefix = prefix

        for letter in ('a', 'b', 'c', 'd', 'e'):
            file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
            file_item.write(letter.encode('ascii') * 10)

            file_item = cls.container.file(
                "%s/seg_upper_%%ff%s" % (prefix, letter))
            file_item.write(letter.upper().encode('ascii') * 10)

        for letter in ('f', 'g', 'h', 'i', 'j'):
            file_item = cls.container2.file("%s/seg_lower%s" %
                                            (prefix, letter))
            file_item.write(letter.encode('ascii') * 10)

        man1 = cls.container.file("man1")
        man1.write(b'man1-contents',
                   hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
                         (cls.container.name, prefix)})

        man2 = cls.container.file("man2")
        man2.write(b'man2-contents',
                   hdrs={"X-Object-Manifest": "%s/%s/seg_upper_%%25ff" %
                         (cls.container.name, prefix)})

        manall = cls.container.file("manall")
        manall.write(b'manall-contents',
                     hdrs={"X-Object-Manifest": "%s/%s/seg" %
                           (cls.container.name, prefix)})

        mancont2 = cls.container.file("mancont2")
        mancont2.write(
            b'mancont2-contents',
            hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
                                       (cls.container2.name, prefix)})


class TestDlo(Base):
    env = TestDloEnv

    def test_get_manifest(self):
        file_item = self.env.container.file('man1')
        file_contents = file_item.read()
        self.assertEqual(
            file_contents,
            b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")

        file_item = self.env.container.file('man2')
        file_contents = file_item.read()
        self.assertEqual(
            file_contents,
            b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")

        file_item = self.env.container.file('manall')
        file_contents = file_item.read()
        self.assertEqual(
            file_contents,
            (b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" +
             b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE"))

    def test_get_manifest_document_itself(self):
        file_item = self.env.container.file('man1')
        file_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(file_contents, b"man1-contents")
        self.assertEqual(file_item.info()['x_object_manifest'],
                         "%s/%s/seg_lower" %
                         (self.env.container.name, self.env.segment_prefix))

    def test_get_range(self):
        file_item = self.env.container.file('man1')
        file_contents = file_item.read(size=25, offset=8)
        self.assertEqual(file_contents, b"aabbbbbbbbbbccccccccccddd")

        file_contents = file_item.read(size=1, offset=47)
        self.assertEqual(file_contents, b"e")

    def test_get_multiple_ranges(self):
        file_item = self.env.container.file('man1')
        file_contents = file_item.read(
            hdrs={'Range': 'bytes=0-4,10-14'})
        self.assert_status(200)  # *not* 206
        self.assertEqual(
            file_contents,
            b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")

    def test_get_range_out_of_range(self):
        file_item = self.env.container.file('man1')

        self.assertRaises(ResponseError, file_item.read, size=7, offset=50)
        self.assert_status(416)
        self.assert_header('content-range', 'bytes */50')

    def test_copy(self):
        # Adding a new segment, copying the manifest, and then deleting the
        # segment proves that the new object is really the concatenated
        # segments and not just a manifest.
        f_segment = self.env.container.file("%s/seg_lowerf" %
                                            (self.env.segment_prefix))
        f_segment.write(b'ffffffffff')
        try:
            man1_item = self.env.container.file('man1')
            man1_item.copy(self.env.container.name, "copied-man1")
        finally:
            # try not to leave this around for other tests to stumble over
            f_segment.delete(tolerate_missing=True)

        file_item = self.env.container.file('copied-man1')
        file_contents = file_item.read()
        self.assertEqual(
            file_contents,
            b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
        # The copied object must not have X-Object-Manifest
        self.assertNotIn("x_object_manifest", file_item.info())

    def test_copy_account(self):
        # dlo use same account and same container only
        acct = self.env.conn.account_name
        # Adding a new segment, copying the manifest, and then deleting the
        # segment proves that the new object is really the concatenated
        # segments and not just a manifest.
        f_segment = self.env.container.file("%s/seg_lowerf" %
                                            (self.env.segment_prefix))
        f_segment.write(b'ffffffffff')
        try:
            man1_item = self.env.container.file('man1')
            man1_item.copy_account(acct,
                                   self.env.container.name,
                                   "copied-man1")
        finally:
            # try not to leave this around for other tests to stumble over
            f_segment.delete(tolerate_missing=True)

        file_item = self.env.container.file('copied-man1')
        file_contents = file_item.read()
        self.assertEqual(
            file_contents,
            b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
        # The copied object must not have X-Object-Manifest
        self.assertNotIn("x_object_manifest", file_item.info())

    def test_copy_manifest(self):
        # Copying the manifest with multipart-manifest=get query string
        # should result in another manifest
        try:
            man1_item = self.env.container.file('man1')
            man1_item.copy(self.env.container.name, "copied-man1",
                           parms={'multipart-manifest': 'get'})

            copied = self.env.container.file("copied-man1")
            copied_contents = copied.read(parms={'multipart-manifest': 'get'})
            self.assertEqual(copied_contents, b"man1-contents")

            copied_contents = copied.read()
            self.assertEqual(
                copied_contents,
                b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
            self.assertEqual(man1_item.info()['x_object_manifest'],
                             copied.info()['x_object_manifest'])
        finally:
            # try not to leave this around for other tests to stumble over
            self.env.container.file("copied-man1").delete(
                tolerate_missing=True)

    def test_dlo_if_match_get(self):
        manifest = self.env.container.file("man1")
        etag = manifest.info()['etag']

        self.assertRaises(ResponseError, manifest.read,
                          hdrs={'If-Match': 'not-%s' % etag})
        self.assert_status(412)

        manifest.read(hdrs={'If-Match': etag})
        self.assert_status(200)

    def test_dlo_if_none_match_get(self):
        manifest = self.env.container.file("man1")
        etag = manifest.info()['etag']

        self.assertRaises(ResponseError, manifest.read,
                          hdrs={'If-None-Match': etag})
        self.assert_status(304)

        manifest.read(hdrs={'If-None-Match': "not-%s" % etag})
        self.assert_status(200)

    def test_dlo_if_match_head(self):
        manifest = self.env.container.file("man1")
        etag = manifest.info()['etag']

        self.assertRaises(ResponseError, manifest.info,
                          hdrs={'If-Match': 'not-%s' % etag})
        self.assert_status(412)

        manifest.info(hdrs={'If-Match': etag})
        self.assert_status(200)

    def test_dlo_if_none_match_head(self):
        manifest = self.env.container.file("man1")
        etag = manifest.info()['etag']

        self.assertRaises(ResponseError, manifest.info,
                          hdrs={'If-None-Match': etag})
        self.assert_status(304)

        manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
        self.assert_status(200)

    def test_dlo_referer_on_segment_container(self):
        if 'username3' not in tf.config:
            raise tf.SkipTest('Requires user 3')
        # First the account2 (test3) should fail
        config2 = tf.config.copy()
        config2['username'] = tf.config['username3']
        config2['password'] = tf.config['password3']
        conn2 = Connection(config2)
        conn2.authenticate()
        headers = {'X-Auth-Token': conn2.storage_token,
                   'Referer': 'http://blah.example.com'}
        dlo_file = self.env.container.file("mancont2")
        self.assertRaises(ResponseError, dlo_file.read,
                          hdrs=headers)
        self.assert_status(403)

        # Now set the referer on the dlo container only
        referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'}
        self.env.container.update_metadata(referer_metadata)

        self.assertRaises(ResponseError, dlo_file.read,
                          hdrs=headers)
        self.assert_status(403)

        # Finally set the referer on the segment container
        self.env.container2.update_metadata(referer_metadata)

        contents = dlo_file.read(hdrs=headers)
        self.assertEqual(
            contents,
            b"ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")

    def test_dlo_post_with_manifest_header(self):
        # verify that performing a POST to a DLO manifest
        # preserves the fact that it is a manifest file.
        # verify that the x-object-manifest header may be updated.

        # create a new manifest for this test to avoid test coupling.
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
        file_item = self.env.container.file(Utils.create_name())
        file_item.write(b'manifest-contents',
                        hdrs={"X-Object-Manifest": x_o_m})

        # sanity checks
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)
        expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # POST a modified x-object-manifest value
        new_x_o_m = x_o_m.rstrip('lower') + 'upper'
        file_item.post({'x-object-meta-foo': 'bar',
                        'x-object-manifest': new_x_o_m})

        # verify that x-object-manifest was updated
        file_item.info()
        resp_headers = [(h.lower(), v)
                        for h, v in file_item.conn.response.getheaders()]
        self.assertIn(('x-object-manifest', str_to_wsgi(new_x_o_m)),
                      resp_headers)
        self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)

        # verify that manifest content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)

        # verify that updated manifest points to new content
        expected_contents = ''.join((c * 10) for c in 'ABCDE').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # Now revert the manifest to point to original segments, including a
        # multipart-manifest=get param just to check that has no effect
        file_item.post({'x-object-manifest': x_o_m},
                       parms={'multipart-manifest': 'get'})

        # verify that x-object-manifest was reverted
        info = file_item.info()
        self.assertIn('x_object_manifest', info)
        self.assertEqual(x_o_m, info['x_object_manifest'])

        # verify that manifest content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)

        # verify that updated manifest points new content
        expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

    def test_dlo_post_without_manifest_header(self):
        # verify that a POST to a DLO manifest object with no
        # x-object-manifest header will cause the existing x-object-manifest
        # header to be lost

        # create a new manifest for this test to avoid test coupling.
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
        file_item = self.env.container.file(Utils.create_name())
        file_item.write(b'manifest-contents',
                        hdrs={"X-Object-Manifest": x_o_m})

        # sanity checks
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)
        expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # POST with no x-object-manifest header
        file_item.post({})

        # verify that existing x-object-manifest was removed
        info = file_item.info()
        self.assertNotIn('x_object_manifest', info)

        # verify that object content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)

        # verify that object is no longer a manifest
        contents = file_item.read(parms={})
        self.assertEqual(b'manifest-contents', contents)

    def test_dlo_post_with_manifest_regular_object(self):
        # verify that performing a POST to a regular object
        # with a manifest header will create a DLO.

        # Put a regular object
        file_item = self.env.container.file(Utils.create_name())
        file_item.write(b'file contents', hdrs={})

        # sanity checks
        file_contents = file_item.read(parms={})
        self.assertEqual(b'file contents', file_contents)

        # get the path associated with man1
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']

        # POST a x-object-manifest value to the regular object
        file_item.post({'x-object-manifest': x_o_m})

        # verify that the file is now a manifest
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'file contents', manifest_contents)
        expected_contents = ''.join([(c * 10) for c in 'abcde']).encode()
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)
        file_item.info()
        resp_headers = [(h.lower(), v)
                        for h, v in file_item.conn.response.getheaders()]
        self.assertIn(('x-object-manifest', str_to_wsgi(x_o_m)), resp_headers)


class TestDloUTF8(Base2, TestDlo):
    pass