path: root/test/suite/test_checkpoint01.py

#!/usr/bin/env python
#
# Public Domain 2014-2015 MongoDB, Inc.
# Public Domain 2008-2014 WiredTiger, Inc.
#
# This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

import wiredtiger, wttest
from helper import key_populate, complex_populate_lsm, simple_populate
from wtscenario import check_scenarios

# test_checkpoint01.py
#    Checkpoint tests
# General checkpoint test: create an object containing sets of data associated
# with a set of checkpoints, then confirm the checkpoint's values are correct,
# including after other checkpoints are dropped.
class test_checkpoint(wttest.WiredTigerTestCase):
    scenarios = check_scenarios([
        ('file', dict(uri='file:checkpoint',fmt='S')),
        ('table', dict(uri='table:checkpoint',fmt='S'))
    ])

    # Each checkpoint has a key range and a "is dropped" flag.
    checkpoints = {
        "checkpoint-1": ((100, 200), 0),
        "checkpoint-2": ((200, 220), 0),
        "checkpoint-3": ((300, 320), 0),
        "checkpoint-4": ((400, 420), 0),
        "checkpoint-5": ((500, 520), 0),
        "checkpoint-6": ((100, 620), 0),
        "checkpoint-7": ((200, 250), 0),
        "checkpoint-8": ((300, 820), 0),
        "checkpoint-9": ((400, 920), 0)
        }

    # Add a set of records for a checkpoint.
    def add_records(self, name):
        cursor = self.session.open_cursor(self.uri, None, "overwrite")
        start, stop = self.checkpoints[name][0]
        for i in range(start, stop+1):
            cursor["%010d KEY------" % i] = ("%010d VALUE " % i) + name
        cursor.close()
        self.checkpoints[name] = (self.checkpoints[name][0], 1)

    # For each checkpoint entry, add/overwrite the specified records, then
    # checkpoint the object, and verify it (which verifies all underlying
    # checkpoints individually).
    def build_file_with_checkpoints(self):
        for checkpoint_name, entry in self.checkpoints.iteritems():
            self.add_records(checkpoint_name)
            self.session.checkpoint("name=" + checkpoint_name)
            
    # Create a dictionary of sorted records a checkpoint should include.
    def list_expected(self, name):
        records = {}
        for checkpoint_name, entry in self.checkpoints.iteritems():
            start, stop = entry[0]
            for i in range(start, stop+1):
                records['%010d KEY------' % i] =\
                    '%010d VALUE ' % i + checkpoint_name
            if name == checkpoint_name:
                break
        return records

    # Create a dictionary of sorted records a checkpoint does include.
    def list_checkpoint(self, name):
        records = {}
        cursor = self.session.open_cursor(self.uri, None, 'checkpoint=' + name)
        while cursor.next() == 0:
            records[cursor.get_key()] = cursor.get_value()
        cursor.close()
        return records

    # For each existing checkpoint entry, verify it contains the records it
    # should, and no other checkpoints exist.
    def check(self):
        # Physically verify the file, including the individual checkpoints.
        self.session.verify(self.uri, None)

        for checkpoint_name, entry in self.checkpoints.iteritems():
            if entry[1] == 0:
                self.assertRaises(wiredtiger.WiredTigerError,
                    lambda: self.session.open_cursor(
                    self.uri, None, "checkpoint=" + checkpoint_name))
            else:
                list_expected = self.list_expected(checkpoint_name)
                list_checkpoint = self.list_checkpoint(checkpoint_name)
                self.assertEqual(list_expected, list_checkpoint)

    # Main checkpoint test driver.
    def test_checkpoint(self):
        # Build a file with a set of checkpoints, and confirm they all have
        # the correct key/value pairs.
        self.session.create(self.uri,
            "key_format=" + self.fmt +\
                ",value_format=S,allocation_size=512,leaf_page_max=512")
        self.build_file_with_checkpoints()
        self.check()

        # Drop a set of checkpoints sequentially, and each time confirm the
        # contents of remaining checkpoints, and that dropped checkpoints
        # don't appear.
        for i in [1,3,7,9]:
            checkpoint_name = 'checkpoint-' + str(i)
            self.session.checkpoint('drop=(' + checkpoint_name + ')')
            self.checkpoints[checkpoint_name] =\
                (self.checkpoints[checkpoint_name][0], 0)
            self.check()

        # Drop remaining checkpoints, all subsequent checkpoint opens should
        # fail.
        self.session.checkpoint("drop=(from=all)")
        for checkpoint_name, entry in self.checkpoints.iteritems():
            self.checkpoints[checkpoint_name] =\
                (self.checkpoints[checkpoint_name][0], 0)
        self.check()


# Check some specific cursor checkpoint combinations.
class test_checkpoint_cursor(wttest.WiredTigerTestCase):
    scenarios = check_scenarios([
        ('file', dict(uri='file:checkpoint',fmt='S')),
        ('table', dict(uri='table:checkpoint',fmt='S'))
    ])

    # Check that you cannot open a checkpoint that doesn't exist.
    def test_checkpoint_dne(self):
        simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
        self.assertRaises(wiredtiger.WiredTigerError,
            lambda: self.session.open_cursor(
            self.uri, None, "checkpoint=checkpoint-1"))
        self.assertRaises(wiredtiger.WiredTigerError,
            lambda: self.session.open_cursor(
            self.uri, None, "checkpoint=WiredTigerCheckpoint"))

    # Check that you can open checkpoints more than once.
    def test_checkpoint_multiple_open(self):
        simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
        self.session.checkpoint("name=checkpoint-1")
        c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
        c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
        c3 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
        c4 = self.session.open_cursor(self.uri, None, None)
        c4.close, c3.close, c2.close, c1.close

        self.session.checkpoint("name=checkpoint-2")
        c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
        c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-2")
        c3 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-2")
        c4 = self.session.open_cursor(self.uri, None, None)
        c4.close, c3.close, c2.close, c1.close

    # Check that you cannot drop a checkpoint if it's in use.
    def test_checkpoint_inuse(self):
        simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
        self.session.checkpoint("name=checkpoint-1")
        self.session.checkpoint("name=checkpoint-2")
        self.session.checkpoint("name=checkpoint-3")
        cursor = self.session.open_cursor(
            self.uri, None, "checkpoint=checkpoint-2")

        # Check creating an identically named checkpoint fails. */
        # Check dropping the specific checkpoint fails.
        # Check dropping all checkpoints fails.
        msg = '/checkpoints cannot be dropped/'
        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
            lambda: self.session.checkpoint("name=checkpoint-2"), msg)
        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
            lambda: self.session.checkpoint("drop=(checkpoint-2)"), msg)
        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
            lambda: self.session.checkpoint("drop=(from=all)"), msg)

        # Check dropping other checkpoints succeeds (which also tests that you
        # can create new checkpoints while other checkpoints are in-use).
        self.session.checkpoint("drop=(checkpoint-1,checkpoint-3)")

        # Close the cursor and repeat the failing commands, they should now
        # succeed.
        cursor.close()
        self.session.checkpoint("name=checkpoint-2")
        self.session.checkpoint("drop=(checkpoint-2)")
        self.session.checkpoint("drop=(from=all)")


# Check that you can checkpoint targets.
class test_checkpoint_target(wttest.WiredTigerTestCase):
    scenarios = check_scenarios([
        ('file', dict(uri='file:checkpoint',fmt='S')),
        ('table', dict(uri='table:checkpoint',fmt='S'))
    ])

    def update(self, uri, value):
        cursor = self.session.open_cursor(uri, None, "overwrite")
        cursor[key_populate(cursor, 10)] = value
        cursor.close()

    def check(self, uri, value):
        cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint-1")
        self.assertEquals(cursor[key_populate(cursor, 10)], value)
        cursor.close()

    def test_checkpoint_target(self):
        # Create 3 objects, change one record to an easily recognizable string.
        uri = self.uri + '1'
        simple_populate(self, uri, 'key_format=' + self.fmt, 100)
        self.update(uri, 'ORIGINAL')
        uri = self.uri + '2'
        simple_populate(self, uri, 'key_format=' + self.fmt, 100)
        self.update(uri, 'ORIGINAL')
        uri = self.uri + '3'
        simple_populate(self, uri, 'key_format=' + self.fmt, 100)
        self.update(uri, 'ORIGINAL')

        # Checkpoint all three objects.
        self.session.checkpoint("name=checkpoint-1")

        # Update all 3 objects, then checkpoint two of the objects with the
        # same checkpoint name.
        self.update(self.uri + '1', 'UPDATE')
        self.update(self.uri + '2', 'UPDATE')
        self.update(self.uri + '3', 'UPDATE')
        target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")'
        self.session.checkpoint("name=checkpoint-1," + target)

        # Confirm the checkpoint has the old value in objects that weren't
        # checkpointed, and the new value in objects that were checkpointed.
        self.check(self.uri + '1', 'UPDATE')
        self.check(self.uri + '2', 'UPDATE')
        self.check(self.uri + '3', 'ORIGINAL')


# Check that you can't write checkpoint cursors.
class test_checkpoint_cursor_update(wttest.WiredTigerTestCase):
    scenarios = check_scenarios([
        ('file-r', dict(uri='file:checkpoint',fmt='r')),
        ('file-S', dict(uri='file:checkpoint',fmt='S')),
        ('table-r', dict(uri='table:checkpoint',fmt='r')),
        ('table-S', dict(uri='table:checkpoint',fmt='S'))
    ])

    def test_checkpoint_cursor_update(self):
        simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
        self.session.checkpoint("name=ckpt")
        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
        cursor.set_key(key_populate(cursor, 10))
        cursor.set_value("XXX")
        self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert())
        self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.remove())
        self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.update())
        cursor.close()


# Check that WiredTigerCheckpoint works as a checkpoint specifier.
class test_checkpoint_last(wttest.WiredTigerTestCase):
    scenarios = check_scenarios([
        ('file', dict(uri='file:checkpoint',fmt='S')),
        ('table', dict(uri='table:checkpoint',fmt='S'))
    ])

    def test_checkpoint_last(self):
        # Create an object, change one record to an easily recognizable string,
        # then checkpoint it and open a cursor, confirming we see the correct
        # value.   Repeat this action, we want to be sure the engine gets the
        # latest checkpoint information each time.
        uri = self.uri
        simple_populate(self, uri, 'key_format=' + self.fmt, 100)

        for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'):
            # Update the object.
            cursor = self.session.open_cursor(uri, None, "overwrite")
            cursor[key_populate(cursor, 10)] = value
            cursor.close()

            # Checkpoint the object.
            self.session.checkpoint()

            # Verify the "last" checkpoint sees the correct value.
            cursor = self.session.open_cursor(
                uri, None, "checkpoint=WiredTigerCheckpoint")
            self.assertEquals(cursor[key_populate(cursor, 10)], value)
            # Don't close the checkpoint cursor, we want it to remain open until
            # the test completes.


# Check we can't use the reserved name as an application checkpoint name.
class test_checkpoint_illegal_name(wttest.WiredTigerTestCase):
    def test_checkpoint_illegal_name(self):
        simple_populate(self, "file:checkpoint", 'key_format=S', 100)
        msg = '/the checkpoint name.*is reserved/'
        for conf in (
            'name=WiredTigerCheckpoint',
            'name=WiredTigerCheckpoint.',
            'name=WiredTigerCheckpointX',
            'drop=(from=WiredTigerCheckpoint)',
            'drop=(from=WiredTigerCheckpoint.)',
            'drop=(from=WiredTigerCheckpointX)',
            'drop=(to=WiredTigerCheckpoint)',
            'drop=(to=WiredTigerCheckpoint.)',
            'drop=(to=WiredTigerCheckpointX)'):
                self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
                    lambda: self.session.checkpoint(conf), msg)
        msg = '/WiredTiger objects should not include grouping/'
        for conf in (
            'name=check{point',
            'name=check\\point'):
                self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
                    lambda: self.session.checkpoint(conf), msg)


# Check we can't name checkpoints that include LSM tables.
class test_checkpoint_lsm_name(wttest.WiredTigerTestCase):
    def test_checkpoint_lsm_name(self):
        complex_populate_lsm(self,
            "table:checkpoint", 'type=lsm,key_format=S', 1000)
        msg = '/object does not support named checkpoints/'
        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
            lambda: self.session.checkpoint("name=ckpt"), msg)


class test_checkpoint_empty(wttest.WiredTigerTestCase):
    scenarios = check_scenarios([
        ('file', dict(uri='file:checkpoint')),
        ('table', dict(uri='table:checkpoint')),
    ])

    # Create an empty file, do one of 4 cases of checkpoint, then verify the
    # checkpoints exist.   The reason for the 4 cases is we must create all
    # checkpoints an application can explicitly reference using a cursor, and
    # I want to test when other types of checkpoints are created before the
    # checkpoint we really need.
    # Case 1: a named checkpoint
    def test_checkpoint_empty_one(self):
        self.session.create(self.uri, "key_format=S,value_format=S")
        self.session.checkpoint('name=ckpt')
        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")

    # Case 2: an internal checkpoint
    def test_checkpoint_empty_two(self):
        self.session.create(self.uri, "key_format=S,value_format=S")
        self.session.checkpoint()
        cursor = self.session.open_cursor(
            self.uri, None, "checkpoint=WiredTigerCheckpoint")

    # Case 3: a named checkpoint, then an internal checkpoint
    def test_checkpoint_empty_three(self):
        self.session.create(self.uri, "key_format=S,value_format=S")
        self.session.checkpoint('name=ckpt')
        self.session.checkpoint()
        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
        cursor = self.session.open_cursor(
            self.uri, None, "checkpoint=WiredTigerCheckpoint")

    # Case 4: an internal checkpoint, then a named checkpoint
    def test_checkpoint_empty_four(self):
        self.session.create(self.uri, "key_format=S,value_format=S")
        self.session.checkpoint()
        self.session.checkpoint('name=ckpt')
        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
        cursor = self.session.open_cursor(
            self.uri, None, "checkpoint=WiredTigerCheckpoint")

    # Check that we can create an empty checkpoint, change the underlying
    # object, checkpoint again, and still see the original empty tree, for
    # both named and unnamed checkpoints.
    def test_checkpoint_empty_five(self):
        self.session.create(self.uri, "key_format=S,value_format=S")
        self.session.checkpoint('name=ckpt')
        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)
        cursor.close()

        cursor = self.session.open_cursor(self.uri, None)
        cursor["key"] = "value"
        self.session.checkpoint()

        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)

    def test_checkpoint_empty_six(self):
        self.session.create(self.uri, "key_format=S,value_format=S")
        self.session.checkpoint()
        cursor = self.session.open_cursor(
            self.uri, None, "checkpoint=WiredTigerCheckpoint")
        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)
        cursor.close()

        cursor = self.session.open_cursor(self.uri, None)
        cursor["key"] = "value"
        self.session.checkpoint('name=ckpt')

        cursor = self.session.open_cursor(
            self.uri, None, "checkpoint=WiredTigerCheckpoint")
        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)


if __name__ == '__main__':
    wttest.run()