diff options
Diffstat (limited to 'src/third_party/wiredtiger/test/suite')
4 files changed, 421 insertions, 5 deletions
diff --git a/src/third_party/wiredtiger/test/suite/test_backup01.py b/src/third_party/wiredtiger/test/suite/test_backup01.py index ff4dd439189..00a3ccaf0f6 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup01.py +++ b/src/third_party/wiredtiger/test/suite/test_backup01.py @@ -30,6 +30,7 @@ import glob import os import shutil import string +import time from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet @@ -163,8 +164,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(ret, wiredtiger.WT_NOTFOUND) self.assertEqual(i, total) - # Test that named checkpoints can't be deleted while backup cursors are - # open, but that normal checkpoints continue to work. + # Test interaction between checkpoints and a backup cursor. def test_checkpoint_delete(self): # You cannot name checkpoints including LSM tables, skip those. self.populate(1) @@ -177,7 +177,8 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): self.objs[0][0], None, "checkpoint=one")) # Confirm opening a backup cursor causes checkpoint to fail if dropping - # a named checkpoint, but does not stop a default checkpoint. + # a named checkpoint created before the backup cursor, but does not stop a + # default checkpoint. cursor = self.session.open_cursor('backup:', None, None) self.session.checkpoint() msg = '/checkpoints cannot be deleted during a hot backup/' @@ -187,7 +188,23 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.checkpoint("name=three,drop=(two)"), msg) self.session.checkpoint() + + # Need to pause a couple seconds; checkpoints that are assigned the same timestamp as + # the backup will be pinned, even if they occur after the backup starts. + time.sleep(2) + self.session.checkpoint("name=four") + self.session.checkpoint("drop=(four)") + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.open_cursor( + self.objs[0][0], None, "checkpoint=four")) + + # Confirm that after closing the backup cursor the original named checkpoint can + # be deleted. cursor.close() + self.session.checkpoint("drop=(two)") + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.open_cursor( + self.objs[0][0], None, "checkpoint=two")) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup14.py b/src/third_party/wiredtiger/test/suite/test_backup14.py index 7a2ec4f427f..c312020bcef 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup14.py +++ b/src/third_party/wiredtiger/test/suite/test_backup14.py @@ -175,9 +175,9 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): shutil.copy(copy_from, copy_to) else: self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) - write_from = newfile + read_from = newfile write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile - rfp = open(write_from, "r+b") + rfp = open(read_from, "r+b") wfp = open(write_to, "w+b") rfp.seek(offset, 0) wfp.seek(offset, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_backup15.py b/src/third_party/wiredtiger/test/suite/test_backup15.py new file mode 100644 index 00000000000..08ddf177ea6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup15.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios +import glob + +# test_backup15.py +# Test cursor backup with a block-based incremental cursor. +class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): + bkp_home = "WT_BLOCK" + counter=0 + conn_config='cache_size=1G,log=(enabled,file_max=100K)' + logmax="100K" + max_iteration=7 + mult=0 + nops=100000 + savefirst=0 + savekey='NOTSET' + uri="table:main" + + dir='backup.dir' # Backup directory name + home_full = "WT_BLOCK_LOG_FULL" + home_incr = "WT_BLOCK_LOG_INCR" + + full_out = "./backup_block_full" + incr_out = "./backup_block_incr" + logpath = "logpath" + new_table=False + initial_backup=False + + pfx = 'test_backup' + # Set the key and value big enough that we modify a few blocks. + bigkey = 'Key' * 100 + bigval = 'Value' * 100 + + # + # Set up all the directories needed for the test. We have a full backup directory for each + # iteration and an incremental backup for each iteration. That way we can compare the full and + # incremental each time through. + # + def setup_directories(self): + for i in range(0, self.max_iteration): + remove_dir = self.home_incr + '.' + str(i) + + create_dir = self.home_incr + '.' + str(i) + '/' + self.logpath + if os.path.exists(remove_dir): + os.remove(remove_dir) + os.makedirs(create_dir) + + if i == 0: + continue + remove_dir = self.home_full + '.' + str(i) + create_dir = self.home_full + '.' + str(i) + '/' + self.logpath + if os.path.exists(remove_dir): + os.remove(remove_dir) + os.makedirs(create_dir) + + def take_full_backup(self): + if self.counter != 0: + hdir = self.home_full + '.' + str(self.counter) + else: + hdir = self.home_incr + + # + # First time through we take a full backup into the incremental directories. Otherwise only + # into the appropriate full directory. + # + buf = None + if self.initial_backup == True: + buf = 'incremental=(granularity=1M,enabled=true,this_id=ID0)' + + cursor = self.session.open_cursor('backup:', None, buf) + while True: + ret = cursor.next() + if ret != 0: + break + newfile = cursor.get_key() + + if self.counter == 0: + # Take a full bakcup into each incremental directory + for i in range(0, self.max_iteration): + copy_from = newfile + # If it is log file, prepend the path. + if ("WiredTigerLog" in newfile): + copy_to = self.home_incr + '.' + str(i) + '/' + self.logpath + else: + copy_to = self.home_incr + '.' + str(i) + shutil.copy(copy_from, copy_to) + else: + copy_from = newfile + # If it is log file, prepend the path. + if ("WiredTigerLog" in newfile): + copy_to = hdir + '/' + self.logpath + else: + copy_to = hdir + + shutil.copy(copy_from, copy_to) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + cursor.close() + + def take_incr_backup(self): + # Open the backup data source for incremental backup. + buf = 'incremental=(src_id="ID' + str(self.counter-1) + '",this_id="ID' + str(self.counter) + '")' + self.pr(buf) + bkup_c = self.session.open_cursor('backup:', None, buf) + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + h = self.home_incr + '.0' + copy_from = newfile + # If it is log file, prepend the path. + if ("WiredTigerLog" in newfile): + copy_to = h + '/' + self.logpath + else: + copy_to = h + + shutil.copy(copy_from, copy_to) + first = True + config = 'incremental=(file=' + newfile + ')' + dup_cnt = 0 + incr_c = self.session.open_cursor(None, bkup_c, config) + + # For each file listed, open a duplicate backup cursor and copy the blocks. + while True: + ret = incr_c.next() + if ret != 0: + break + incrlist = incr_c.get_keys() + offset = incrlist[0] + size = incrlist[1] + curtype = incrlist[2] + # 1 is WT_BACKUP_FILE + # 2 is WT_BACKUP_RANGE + self.assertTrue(curtype == 1 or curtype == 2) + if curtype == 1: + if first == True: + h = self.home_incr + '.' + str(self.counter) + first = False + + copy_from = newfile + if ("WiredTigerLog" in newfile): + copy_to = h + '/' + self.logpath + else: + copy_to = h + shutil.copy(copy_from, copy_to) + else: + self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) + read_from = newfile + if self.counter > 0: + old_to = self.home_incr + '.' + str(self.counter - 1) + '/' + newfile + else: + old_to = newfile + write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile + write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile + rfp = open(read_from, "r+b") + self.pr('RANGE CHECK file ' + old_to + ' offset ' + str(offset) + ' len ' + str(size)) + rfp2 = open(old_to, "r+b") + rfp.seek(offset, 0) + rfp2.seek(offset, 0) + buf = rfp.read(size) + buf2 = rfp2.read(size) + self.assertNotEqual(buf, buf2) + wfp = open(write_to, "w+b") + wfp.seek(offset, 0) + wfp.write(buf) + rfp.close() + rfp2.close() + wfp.close() + dup_cnt += 1 + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + incr_c.close() + + # For each file, we want to copy the file into each of the later incremental directories + for i in range(self.counter, self.max_iteration): + h = self.home_incr + '.' + str(i) + copy_from = newfile + if ("WiredTigerLog" in newfile): + copy_to = h + '/' + self.logpath + else: + copy_to = h + shutil.copy(copy_from, copy_to) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + bkup_c.close() + + def compare_backups(self, t_uri): + # + # Run wt dump on full backup directory + # + full_backup_out = self.full_out + '.' + str(self.counter) + home_dir = self.home_full + '.' + str(self.counter) + if self.counter == 0: + home_dir = self.home + + self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=full_backup_out) + # + # Run wt dump on incremental backup directory + # + incr_backup_out = self.incr_out + '.' + str(self.counter) + home_dir = self.home_incr + '.' + str(self.counter) + self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=incr_backup_out) + + self.assertEqual(True, + compare_files(self, full_backup_out, incr_backup_out)) + + # + # Add data to the given uri. + # + def add_data(self, uri): + c = self.session.open_cursor(uri, None, None) + # The first time we want to add in a lot of data. Then after that we want to + # rapidly change a single key to create a hotspot in one block. + if self.savefirst < 2: + nops = self.nops + else: + nops = self.nops // 10 + for i in range(0, nops): + num = i + (self.mult * nops) + if self.savefirst >= 2: + key = self.savekey + else: + key = str(num) + self.bigkey + str(num) + val = str(num) + self.bigval + str(num) + c[key] = val + if self.savefirst == 0: + self.savekey = key + self.savefirst += 1 + c.close() + + # Increase the multiplier so that later calls insert unique items. + self.mult += 1 + # Increase the counter so that later backups have unique ids. + if self.initial_backup == False: + self.counter += 1 + + # + # This function will add records to the table (table:main), take incremental/full backups and + # validate the backups. + # + def add_data_validate_backups(self): + self.pr('Adding initial data') + self.initial_backup = True + self.add_data(self.uri) + self.take_full_backup() + self.initial_backup = False + self.session.checkpoint() + + # Each call now to take a full backup will make a copy into a full directory. Then + # each incremental will take an incremental backup and we can compare them. + self.add_data(self.uri) + self.session.checkpoint() + self.take_full_backup() + self.take_incr_backup() + self.compare_backups(self.uri) + + self.add_data(self.uri) + self.session.checkpoint() + self.take_incr_backup() + self.take_full_backup() + self.compare_backups(self.uri) + + self.add_data(self.uri) + self.session.checkpoint() + self.take_full_backup() + self.take_incr_backup() + self.compare_backups(self.uri) + + self.add_data(self.uri) + self.take_incr_backup() + self.take_full_backup() + self.compare_backups(self.uri) + + def test_backup15(self): + os.mkdir(self.bkp_home) + self.home = self.bkp_home + self.session.create(self.uri, "key_format=S,value_format=S") + + self.setup_directories() + + self.pr('*** Add data, checkpoint, take backups and validate ***') + self.add_data_validate_backups() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint05.py b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py new file mode 100644 index 00000000000..58af3003a60 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_checkpoint05.py +# Verify that we don't accumulate a lot of checkpoints while a backup +# cursor is open. WiredTiger checkpoints created after the backup cursor +# should get deleted as usual. + +import time +import wiredtiger, wttest + +class test_checkpoint05(wttest.WiredTigerTestCase): + conn_config = 'create,cache_size=100MB,log=(archive=false,enabled=true,file_max=100K)' + + def count_checkpoints(self): + metadata_cursor = self.session.open_cursor('metadata:', None, None) + + nckpt = 0 + while metadata_cursor.next() == 0: + key = metadata_cursor.get_key() + value = metadata_cursor[key] + nckpt = nckpt + value.count("WiredTigerCheckpoint") + metadata_cursor.close() + return nckpt + + def test_checkpoints_during_backup(self): + self.uri = 'table:ckpt05' + self.session.create(self.uri, 'key_format=i,value_format=i') + + # Setup: Insert some data and checkpoint it + cursor = self.session.open_cursor(self.uri, None) + for i in range(16): + cursor[i] = i + self.session.checkpoint(None) + + # Create backup and check how many checkpoints we have. + backup_cursor = self.session.open_cursor('backup:', None, None) + initial_count = self.count_checkpoints() + + # Checkpoints created immediately after a backup cursor may get pinned. + # Pause to avoid this. + time.sleep(2) + + # Take a bunch of checkpoints. + for i in range (50): + self.session.checkpoint('force=true') + cursor.close() + + # There may be a few more checkpoints than when we opened the + # backup cursor, but not too many more. The factor of three + # is generous. But if WT isn't deleting checkpoints there would + # be about 30x more checkpoints here. + final_count = self.count_checkpoints() + self.assertTrue (final_count < initial_count * 3) + + self.session.close() + +if __name__ == '__main__': + wttest.run() |