diff options
Diffstat (limited to 'src/third_party/wiredtiger/test/suite')
3 files changed, 147 insertions, 22 deletions
diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py index 0e6e2bce286..98042792da9 100755 --- a/src/third_party/wiredtiger/test/suite/hook_tiered.py +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -331,6 +331,7 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): "test_truncate16.test_truncate16", "test_truncate18.test_truncate18", "test_truncate15.test_truncate15", + "test_truncate19.test_truncate19", "test_txn22.test_corrupt_meta", "test_verbose01.test_verbose_single", "test_verbose02.test_verbose_single", diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py index cd625ae8954..245637688f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_search_near01.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -93,18 +93,18 @@ class test_search_near01(wttest.WiredTigerTestCase): cursor2.set_key('aa') cursor2.search_near() - skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) - # This should be equal to roughly key_count * 2 as we're going to traverse the whole + skip_count = self.get_stat(stat.conn.cursor_next_skip_total) + # This should be equal to roughly key_count - 1 as we're going to traverse the whole # range forward, and then the whole range backwards. - self.assertGreater(skip_count, key_count * 2) + self.assertEqual(skip_count, key_count - 1) cursor2.reconfigure("prefix_search=true") cursor2.set_key('aa') self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) - prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) - # We should've skipped ~26 here as we're only looking at the "aa" range. - self.assertGreaterEqual(prefix_skip_count - skip_count, 26) + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_total) + self.get_stat(stat.conn.cursor_prev_skip_total) + # We should've skipped ~26 - 1 here as we're only looking at the "aa" range. + self.assertGreaterEqual(prefix_skip_count - skip_count, 25) skip_count = prefix_skip_count # The prefix code will have come into play at once as we walked to "aba". The prev @@ -117,7 +117,7 @@ class test_search_near01(wttest.WiredTigerTestCase): self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) # Assert it to have only incremented the skipped statistic ~26 times. - prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_total) + self.get_stat(stat.conn.cursor_prev_skip_total) self.assertGreaterEqual(prefix_skip_count - skip_count, 26) skip_count = prefix_skip_count @@ -136,7 +136,7 @@ class test_search_near01(wttest.WiredTigerTestCase): # fail. # # It should be closer to key_count * 2 but this an approximation. - prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_total) + self.get_stat(stat.conn.cursor_prev_skip_total) self.assertGreaterEqual(prefix_skip_count - skip_count, key_count) # This test aims to simulate a unique index insertion. @@ -192,7 +192,7 @@ class test_search_near01(wttest.WiredTigerTestCase): cursor2.set_key('cc') cursor2.search_near() - skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + skip_count = self.get_stat(stat.conn.cursor_next_skip_total) # This should be slightly greater than key_count as we're going to traverse most of the # range forwards. self.assertGreater(skip_count, key_count) @@ -214,7 +214,7 @@ class test_search_near01(wttest.WiredTigerTestCase): uri = 'table:test_row_search' self.session.create(uri, 'key_format=u,value_format=u') cursor = self.session.open_cursor(uri) - expect_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + expect_count = self.get_stat(stat.conn.cursor_next_skip_total) session2 = self.conn.open_session() l = "abcdefghijklmnopqrstuvwxyz" # Insert keys a -> z, except c @@ -258,7 +258,7 @@ class test_search_near01(wttest.WiredTigerTestCase): cursor.search_near() self.session.commit_transaction() expect_count += 1 - skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + skip_count = self.get_stat(stat.conn.cursor_next_skip_total) self.assertEqual(skip_count, expect_count) # Test a basic prepared scenario. @@ -272,13 +272,13 @@ class test_search_near01(wttest.WiredTigerTestCase): l = "abcdefghijklmnopqrstuvwxyz" session2.begin_transaction() - key_count = 26*26 - # Insert 'cc' self.session.begin_transaction() cursor['cc'] = 'cc' self.session.commit_transaction() + key_count = 1 + # Prepare keys aa -> zz self.session.begin_transaction() for i in range (0, 26): @@ -286,6 +286,7 @@ class test_search_near01(wttest.WiredTigerTestCase): continue for j in range (0, 26): cursor[l[i] + l[j]] = l[i] + l[j] + key_count += 1 self.session.prepare_transaction('prepare_timestamp=2') @@ -296,23 +297,23 @@ class test_search_near01(wttest.WiredTigerTestCase): cursor3.search() cursor3.reset() - # Search near for the "aa" part of the range. + # Search near for the "c" key. cursor2 = session2.open_cursor(uri) cursor2.set_key('c') self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) - skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) + skip_count = self.get_stat(stat.conn.cursor_next_skip_total, session2) # This should be equal to roughly key_count as we're going to traverse the whole - # range forwards. - self.assertGreater(skip_count, key_count) + # range forwards. Not including 'a' and 'b'. + self.assertGreaterEqual(skip_count, key_count - 2*26) cursor2.reconfigure("prefix_search=true") cursor2.set_key('c') self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) - prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_total, session2) # We expect to traverse one entry and have a buffer to account for anomalies. - self.assertEqual(prefix_skip_count - skip_count, 3) + self.assertEqual(prefix_skip_count - skip_count, 1) skip_count = prefix_skip_count # We early exit here as "cc" is not the last key. @@ -324,13 +325,17 @@ class test_search_near01(wttest.WiredTigerTestCase): cursor4.reconfigure("prefix_search=true") cursor4.set_key('c') self.assertEqual(cursor4.search_near(), 1) - prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_total, session2) # We expect to traverse one entry and have a buffer to account for anomalies. - self.assertEqual(prefix_skip_count - skip_count, 2) + # self.assertEqual(prefix_skip_count - skip_count, 2) + # We expect to not skip any entries and return 'cc' + self.assertEqual(prefix_skip_count - skip_count, 0) + self.assertEqual(cursor4.get_key(), b'cc') skip_count = prefix_skip_count cursor4.reconfigure("prefix_search=false") cursor4.set_key('c') ret = cursor4.search_near() self.assertTrue(ret == -1 or ret == 1) - self.assertEqual(self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) - skip_count, 2) + # We expect to not skip any entries and return 'cc' + self.assertEqual(self.get_stat(stat.conn.cursor_next_skip_total, session2) - skip_count, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate19.py b/src/third_party/wiredtiger/test/suite/test_truncate19.py new file mode 100644 index 00000000000..1afda92e738 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_truncate19.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +import os, wttest, suite_random +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_truncate19.py +# +# Test to mimic oplog workload in MongoDB. Ensure the deleted pages are +# cleaned up on disk and we are not using excessive disk space. +class test_truncate19(wttest.WiredTigerTestCase): + conn_config = 'statistics=(all)' + + format_values = [ + ('string_row', dict(key_format='S', value_format='S')), + ] + scenarios = make_scenarios(format_values) + + def append_rows(self, uri, ds, start_row, nrows, value): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + for i in range(start_row, start_row + nrows + 1): + cursor[ds.key(i)] = value + if i % 2 == 0: + self.session.commit_transaction() + self.session.begin_transaction() + self.session.commit_transaction() + cursor.close() + + def do_truncate(self, ds, start_row, nrows): + self.session.begin_transaction() + hicursor = self.session.open_cursor(ds.uri) + hicursor.set_key(ds.key(start_row + nrows)) + self.session.truncate(None, None, hicursor, None) + self.session.commit_transaction() + + def test_truncate19(self): + uri = 'table:oplog' + nrows = 1000000 + + # Create a table. + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format) + ds.populate() + ds_dummy = SimpleDataSet(self, 'table:dummy', 0, key_format=self.key_format, value_format=self.value_format) + ds_dummy.populate() + + value_a = "aaaaa" * 100 + + # Write some data + self.append_rows(uri, ds, 1, nrows, value_a) + self.session.checkpoint() + + # Reopen the database. + self.reopen_conn() + + # Session for checkpoint + session2 = self.conn.open_session() + # Session for long running transaction, to make truncate not globally visible + session3 = self.conn.open_session() + + trunc_rows = 0 + start_num = 1 + end_num = nrows + for i in range(1, 50): + # Start a long running transaction + session3.begin_transaction() + trunc_rows = 10000 + + self.do_truncate(ds, start_num, trunc_rows) + + # Check stats to make sure we fast-deleted at least one page. + stat_cursor = self.session.open_cursor('statistics:', None, None) + fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] + + self.assertGreater(fastdelete_pages, 0) + + # Take a checkpoint. + session2.checkpoint() + # Ensure the datasize is smaller than 600M + self.assertGreater(600000000, os.path.getsize("oplog.wt")) + session3.rollback_transaction() + + self.append_rows(uri, ds, end_num, trunc_rows, value_a) + + end_num = end_num + trunc_rows + start_num = start_num + trunc_rows + + session2.checkpoint() + # Ensure the datasize is smaller than 600M + self.assertGreater(600000000, os.path.getsize("oplog.wt")) + +if __name__ == '__main__': + wttest.run() |