diff options
author | Luke Chen <luke.chen@mongodb.com> | 2022-09-26 08:27:13 +1000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-09-25 23:02:16 +0000 |
commit | eb7e2ad19725cbf2a3024e1e4362535509a3491a (patch) | |
tree | ba0a0f375814aa922f27cf3eb18ea24c513179b5 /src/third_party | |
parent | b463bf5448847019639ff76a70e658dd6837bda7 (diff) | |
download | mongo-eb7e2ad19725cbf2a3024e1e4362535509a3491a.tar.gz |
Import wiredtiger: 7739da50a2c6a51a044915e01bf20cd33bc28eef from branch mongodb-master
ref: 8670ce2dec..7739da50a2
for: 6.2.0-rc0
WT-9742 Add timestamp support to Simple and Complex data sets in Python tests. (#8286)
Diffstat (limited to 'src/third_party')
26 files changed, 393 insertions, 117 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 1669bc8c012..9da754bc186 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "8670ce2dec70063c06becb0c8b060e8c559e0b1b" + "commit": "7739da50a2c6a51a044915e01bf20cd33bc28eef" } diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py index 429bee56529..1d64644e549 100755 --- a/src/third_party/wiredtiger/test/suite/hook_tiered.py +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -55,8 +55,7 @@ # from __future__ import print_function -import os, sys, wthooks -import unittest +import os, sys, unittest, wthooks, wttimestamp from wttest import WiredTigerTestCase # These are the hook functions that are run when particular APIs are called. @@ -164,6 +163,13 @@ def connection_close_replace(orig_connection_close, connection_self, config): ret = orig_connection_close(connection_self, config) return ret +# Called to replace Connection.open_session +def connection_open_session_replace(orig_connection_open_session, connection_self, config): + ret_session = orig_connection_open_session(connection_self, config) + ret_session._connection = connection_self + ret_session._has_transaction = False + return ret_session + # Called to replace Session.checkpoint. # We add a call to flush_tier after the checkpoint to make sure we are exercising tiered # functionality. @@ -178,6 +184,24 @@ def session_checkpoint_replace(orig_session_checkpoint, session_self, config): ' Calling flush_tier() after checkpoint') return session_self.flush_tier(None) +# Called to replace Session.begin_transaction +def session_begin_transaction_replace(orig_session_begin_transaction, session_self, config): + ret = orig_session_begin_transaction(session_self, config) + session_self._has_transaction = True + return ret + +# Called to replace Session.commit_transaction +def session_commit_transaction_replace(orig_session_commit_transaction, session_self, config): + ret = orig_session_commit_transaction(session_self, config) + session_self._has_transaction = False + return ret + +# Called to replace Session.rollback_transaction +def session_rollback_transaction_replace(orig_session_rollback_transaction, session_self, config): + ret = orig_session_rollback_transaction(session_self, config) + session_self._has_transaction = False + return ret + # Called to replace Session.compact def session_compact_replace(orig_session_compact, session_self, uri, config): # Compact isn't implemented for tiered tables. Only call it if this can't be the uri @@ -218,7 +242,9 @@ def session_open_cursor_replace(orig_session_open_cursor, session_self, uri, dup if uri != None and uri.startswith("backup:"): testcase = WiredTigerTestCase.currentTestCase() testcase.skipTest("backup on tiered tables not yet implemented") - return orig_session_open_cursor(session_self, uri, dupcursor, config) + ret_cursor = orig_session_open_cursor(session_self, uri, dupcursor, config) + ret_cursor._session = session_self + return ret_cursor # Called to replace Session.rename def session_rename_replace(orig_session_rename, session_self, uri, newuri, config): @@ -263,6 +289,14 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): # Override some platform APIs self.platform_api = TieredPlatformAPI() + # This hook plays with timestamps, indirectly by modifying the behavior of the *DataSet classes. + # Here we declare our use of timestamp code, so that tests that have their own notion of + # timestamps can be skipped when running with this hook. + def uses(self, use_list): + if "timestamp" in use_list: + return True + return False + # Is this test one we should skip? def skip_test(self, test): # Skip any test that contains one of these strings as a substring @@ -367,6 +401,22 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): self.Connection['close'] = (wthooks.HOOK_REPLACE, lambda s, config=None: connection_close_replace(orig_connection_close, s, config)) + orig_connection_open_session = self.Connection['open_session'] + self.Connection['open_session'] = (wthooks.HOOK_REPLACE, lambda s, config=None: + connection_open_session_replace(orig_connection_open_session, s, config)) + + orig_session_begin_transaction = self.Session['begin_transaction'] + self.Session['begin_transaction'] = (wthooks.HOOK_REPLACE, lambda s, config=None: + session_begin_transaction_replace(orig_session_begin_transaction, s, config)) + + orig_session_commit_transaction = self.Session['commit_transaction'] + self.Session['commit_transaction'] = (wthooks.HOOK_REPLACE, lambda s, config=None: + session_commit_transaction_replace(orig_session_commit_transaction, s, config)) + + orig_session_rollback_transaction = self.Session['rollback_transaction'] + self.Session['rollback_transaction'] = (wthooks.HOOK_REPLACE, lambda s, config=None: + session_rollback_transaction_replace(orig_session_rollback_transaction, s, config)) + orig_session_compact = self.Session['compact'] self.Session['compact'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_compact_replace(orig_session_compact, s, uri, config)) @@ -395,6 +445,12 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): # Override some platform APIs for this hook. class TieredPlatformAPI(wthooks.WiredTigerHookPlatformAPI): + def setUp(self): + self._timestamp = wttimestamp.WiredTigerTimeStamp() + + def tearDown(self): + pass + def tableExists(self, name): for i in range(1, 9): tablename = name + "-000000000{}.wtobj".format(i) @@ -408,6 +464,11 @@ class TieredPlatformAPI(wthooks.WiredTigerHookPlatformAPI): else: return wthooks.DefaultPlatformAPI.initialFileName(uri) + # By default, there is no timestamping by the data set classes. + def getTimestamp(self): + return self._timestamp + + # Every hook file must have a top level initialize function, # returning a list of WiredTigerHook objects. def initialize(arg): diff --git a/src/third_party/wiredtiger/test/suite/test_bug008.py b/src/third_party/wiredtiger/test/suite/test_bug008.py index 5b04cbffb8a..e10c7c40b2f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_bug008.py +++ b/src/third_party/wiredtiger/test/suite/test_bug008.py @@ -54,7 +54,7 @@ class test_bug008(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 0, key_format=self.key_format, value_format=self.value_format) ds.create() - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) # Search for a record past the end of the table, which should fail. cursor.set_key(ds.key(100)) @@ -75,7 +75,7 @@ class test_bug008(wttest.WiredTigerTestCase): self.reopen_conn() # Open a cursor. - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) # Search for a record at the end of the table, which should succeed. cursor.set_key(ds.key(100)) @@ -114,7 +114,7 @@ class test_bug008(wttest.WiredTigerTestCase): # Set up deleted records before and after a set of duplicate records, # and make sure search/search-near returns the correct record. - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) for i in range(20, 100): cursor[ds.key(i)] = '=== IDENTICAL VALUE ===' for i in range(15, 25): @@ -130,7 +130,7 @@ class test_bug008(wttest.WiredTigerTestCase): self.reopen_conn() # Open a cursor. - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) # Search-near for a record in the deleted set before the duplicate set, # which should succeed, returning the first record in the duplicate set. @@ -154,7 +154,7 @@ class test_bug008(wttest.WiredTigerTestCase): # Delete a range of records. for i in range(5, 10): - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) @@ -166,7 +166,7 @@ class test_bug008(wttest.WiredTigerTestCase): # range), as well as some new records after the end. Put the updates in # a separate transaction so they're invisible to another cursor. self.session.begin_transaction() - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) for i in range(5, 10): cursor[ds.key(i)] = ds.value(i + 1000) for i in range(30, 40): @@ -176,7 +176,7 @@ class test_bug008(wttest.WiredTigerTestCase): # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None, session=s) # Search for an existing record in the deleted range, should not find # it. @@ -265,20 +265,20 @@ class test_bug008(wttest.WiredTigerTestCase): self.reopen_conn() # Add some additional visible records. - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) for i in range(100, 120): cursor[ds.key(i)] = ds.value(i) cursor.close() # Begin a transaction, and add some additional records. self.session.begin_transaction() - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) for i in range(120, 140): cursor[ds.key(i)] = ds.value(i) # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None, session=s) # Search for an invisible record. cursor.set_key(ds.key(130)) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py index 89da3973dbe..e8c08cbd48e 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py @@ -295,7 +295,7 @@ class test_checkpoint_last(wttest.WiredTigerTestCase): for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'): # Update the object. - cursor = self.session.open_cursor(uri, None, "overwrite") + cursor = ds.open_cursor(uri, None, "overwrite") cursor[ds.key(10)] = value cursor.close() @@ -303,7 +303,7 @@ class test_checkpoint_last(wttest.WiredTigerTestCase): self.session.checkpoint() # Verify the "last" checkpoint sees the correct value. - cursor = self.session.open_cursor( + cursor = ds.open_cursor( uri, None, "checkpoint=WiredTigerCheckpoint") self.assertEquals(cursor[ds.key(10)], value) # Don't close the checkpoint cursor, we want it to remain open until diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint09.py b/src/third_party/wiredtiger/test/suite/test_checkpoint09.py index ae432b01909..791ccbbbcc8 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint09.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint09.py @@ -86,6 +86,7 @@ class test_checkpoint09(wttest.WiredTigerTestCase): s.rollback_transaction() evict_cursor.close() + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_checkpoint09(self): uri = 'table:ckpt09' nrows = 1000 diff --git a/src/third_party/wiredtiger/test/suite/test_cursor06.py b/src/third_party/wiredtiger/test/suite/test_cursor06.py index 0c21aca771e..88364288f83 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor06.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor06.py @@ -62,12 +62,13 @@ class test_cursor06(wttest.WiredTigerTestCase): cursor.set_key(self.ds.key(10)) cursor.set_value(self.ds.value(10)) + @wttest.skip_for_hook("tiered", "crashes on final connection close") # FIXME-WT-9809 def test_reconfigure_overwrite(self): uri = self.type + self.name for open_config in (None, "overwrite=0", "overwrite=1"): self.session.drop(uri, "force") self.populate(uri) - cursor = self.session.open_cursor(uri, None, open_config) + cursor = self.ds.open_cursor(uri, None, open_config) if open_config != "overwrite=0": self.set_kv(cursor) cursor.insert() @@ -86,7 +87,7 @@ class test_cursor06(wttest.WiredTigerTestCase): for open_config in (None, "readonly=0", "readonly=1"): self.session.drop(uri, "force") self.populate(uri) - cursor = self.session.open_cursor(uri, None, open_config) + cursor = self.ds.open_cursor(uri, None, open_config) msg = '/Unsupported cursor/' if open_config == "readonly=1": self.set_kv(cursor) @@ -100,7 +101,7 @@ class test_cursor06(wttest.WiredTigerTestCase): def test_reconfigure_invalid(self): uri = self.type + self.name self.populate(uri) - c = self.session.open_cursor(uri, None, None) + c = self.ds.open_cursor(uri, None, None) c.reconfigure("overwrite=1") msg = '/Invalid argument/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_cursor09.py b/src/third_party/wiredtiger/test/suite/test_cursor09.py index a7961b1ae82..d53ea2f2106 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor09.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor09.py @@ -57,7 +57,7 @@ class test_cursor09(wttest.WiredTigerTestCase): ds = self.dataset(self, uri, 100, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() - cursor = self.session.open_cursor(uri, None, None) + cursor = ds.open_cursor(uri, None, None) cursor[ds.key(10)] = ds.value(10) msg = '/requires key be set/' self.assertRaisesWithMessage( diff --git a/src/third_party/wiredtiger/test/suite/test_cursor11.py b/src/third_party/wiredtiger/test/suite/test_cursor11.py index 67be17fda19..75d933081f1 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor11.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor11.py @@ -72,7 +72,7 @@ class test_cursor11(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 50, key_format=self.keyfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor() c.set_key(ds.key(25)) self.assertEquals(c.search(), 0) @@ -94,7 +94,7 @@ class test_cursor11(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 50, key_format=self.keyfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor() c.set_key(ds.key(25)) c.remove() @@ -115,7 +115,7 @@ class test_cursor11(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 50, key_format=self.keyfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor() c.set_key(ds.key(25)) self.assertEquals(c.search(), 0) @@ -137,7 +137,7 @@ class test_cursor11(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 50, key_format=self.keyfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor() c.set_key(ds.key(25)) c.set_value(ds.value(300)) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor12.py b/src/third_party/wiredtiger/test/suite/test_cursor12.py index 6e3d418a4c5..614bf5c713c 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor12.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor12.py @@ -218,7 +218,7 @@ class test_cursor12(wttest.WiredTigerTestCase): # apply modifications in order, # confirm the final state row = 10 - c = self.session.open_cursor(self.uri, None) + c = ds.open_cursor() for i in self.list: c.set_key(ds.key(row)) c.set_value(self.make_value(i['o'])) @@ -251,7 +251,7 @@ class test_cursor12(wttest.WiredTigerTestCase): # For each test in the list: # confirm the final state is there. row = 10 - c = self.session.open_cursor(self.uri, None) + c = ds.open_cursor() for i in self.list: c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) @@ -268,7 +268,7 @@ class test_cursor12(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() - c = self.session.open_cursor(self.uri, None) + c = ds.open_cursor() c.set_key(ds.key(10)) msg = '/not supported/' @@ -339,12 +339,13 @@ class test_cursor12(wttest.WiredTigerTestCase): self.modify_confirm(ds, False) # Check that we can perform a large number of modifications to a record. + @wttest.skip_for_hook("tiered", "crashes on commit_transaction or connection close") # FIXME-WT-9809 def test_modify_many(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() - c = self.session.open_cursor(self.uri, None) + c = ds.open_cursor() self.session.begin_transaction("isolation=snapshot") c.set_key(ds.key(10)) orig = self.make_value('abcdefghijklmnopqrstuvwxyz') @@ -371,7 +372,7 @@ class test_cursor12(wttest.WiredTigerTestCase): self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() - c = self.session.open_cursor(self.uri, None) + c = ds.open_cursor() c.set_key(ds.key(10)) self.assertEquals(c.remove(), 0) @@ -396,7 +397,7 @@ class test_cursor12(wttest.WiredTigerTestCase): self.session.begin_transaction("isolation=snapshot") # Insert a new record. - c = self.session.open_cursor(self.uri, None) + c = ds.open_cursor() c.set_key(ds.key(30)) c.set_value(ds.value(30)) self.assertEquals(c.insert(), 0) @@ -411,7 +412,7 @@ class test_cursor12(wttest.WiredTigerTestCase): # Test that another transaction cannot modify our uncommitted record. xs = self.conn.open_session() - xc = xs.open_cursor(self.uri, None) + xc = ds.open_cursor(session = xs) xs.begin_transaction("isolation=snapshot") xc.set_key(ds.key(30)) xc.set_value(ds.value(30)) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py index cf186131091..84cfc4d7c27 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor17.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor17.py @@ -60,11 +60,12 @@ class test_cursor17(wttest.WiredTigerTestCase): self.ds = self.dataset(self, self.type + self.tablename, rownum, key_format=self.keyformat) self.ds.populate() + @wttest.skip_for_hook("tiered", "fails assertion 99") # FIXME-WT-9809 def test_globally_deleted_key(self): self.populate(100) # Delete the largest key. - cursor = self.session.open_cursor(self.type + self.tablename, None) + cursor = self.ds.open_cursor(self.type + self.tablename, None) self.session.begin_transaction() cursor.set_key(100) self.assertEqual(cursor.remove(), 0) @@ -95,7 +96,7 @@ class test_cursor17(wttest.WiredTigerTestCase): self.session.rollback_transaction() # Use evict cursor to evict the key from memory. - evict_cursor = self.session.open_cursor(self.type + self.tablename, None, "debug=(release_evict)") + evict_cursor = self.ds.open_cursor(self.type + self.tablename, None, "debug=(release_evict)") evict_cursor.set_key(100) if self.valueformat != '8t': self.assertEquals(evict_cursor.search(), wiredtiger.WT_NOTFOUND) @@ -223,6 +224,7 @@ class test_cursor17(wttest.WiredTigerTestCase): self.assertEquals(cursor.largest_key(), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() + @wttest.prevent(["timestamp"]) # this test uses timestamps, hooks should not def test_fast_truncate(self): self.populate(100) @@ -248,6 +250,7 @@ class test_cursor17(wttest.WiredTigerTestCase): self.assertEqual(cursor.get_key(), 100) self.session.rollback_transaction() + @wttest.prevent(["timestamp"]) # this test uses timestamps, hooks should not def test_slow_truncate(self): self.populate(100) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index 3b64532c036..ac9fd07a8ee 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -103,7 +103,7 @@ class test_cursor_random(wttest.WiredTigerTestCase): # Assert we only see 20% matches. We expect to see less than that, but we don't want # to chase random test failures, either. - cursor = self.session.open_cursor(uri, None, self.config) + cursor = ds.open_cursor(uri, None, self.config) list=[] for i in range(1,100): self.assertEqual(cursor.next(), 0) @@ -130,7 +130,7 @@ class test_cursor_random(wttest.WiredTigerTestCase): # Assert we only see 20% matches. We expect to see less than that, but we don't want # to chase random test failures, either. - cursor = self.session.open_cursor(uri, None, self.config) + cursor = ds.open_cursor(uri, None, self.config) list=[] for i in range(1, 100): self.assertEqual(cursor.next(), 0) @@ -157,15 +157,15 @@ class test_cursor_random(wttest.WiredTigerTestCase): # Close the connection so everything is forced to disk. self.reopen_conn() - start = self.session.open_cursor(uri, None) + start = ds.open_cursor(uri, None) start.set_key(ds.key(10)) - end = self.session.open_cursor(uri, None) + end = ds.open_cursor(uri, None) end.set_key(ds.key(10000-10)) - self.session.truncate(None, start, end, None) + ds.truncate(None, start, end, None) self.assertEqual(start.close(), 0) self.assertEqual(end.close(), 0) - cursor = self.session.open_cursor(uri, None, self.config) + cursor = ds.open_cursor(uri, None, self.config) for i in range(1,10): self.assertEqual(cursor.next(), 0) @@ -180,9 +180,9 @@ class test_cursor_random(wttest.WiredTigerTestCase): # Close the connection so everything is forced to disk. self.reopen_conn() - self.session.truncate(uri, None, None, None) + ds.truncate(uri, None, None, None) - cursor = self.session.open_cursor(uri, None, self.config) + cursor = ds.open_cursor(uri, None, self.config) for i in range(1,10): self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_flcs01.py b/src/third_party/wiredtiger/test/suite/test_flcs01.py index 4e5f90989b6..fae21b7e02c 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_flcs01.py +++ b/src/third_party/wiredtiger/test/suite/test_flcs01.py @@ -41,8 +41,8 @@ class test_flcs01(wttest.WiredTigerTestCase): conn_config = 'in_memory=false' # Evict the page to force reconciliation. - def evict(self, uri, key, check_value): - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + def evict(self, ds, uri, key, check_value): + evict_cursor = ds.open_cursor(uri, None, "debug=(release_evict)") self.session.begin_transaction() v = evict_cursor[key] self.assertEqual(v, check_value) @@ -105,6 +105,7 @@ class test_flcs01(wttest.WiredTigerTestCase): self.check_prev(cursor, k, 0) self.session.rollback_transaction() + @wttest.skip_for_hook("tiered", "crashes in evict function, during cursor reset") # FIXME-WT-9809 def test_flcs(self): uri = "table:test_flcs01" nrows = 44 @@ -120,7 +121,7 @@ class test_flcs01(wttest.WiredTigerTestCase): appendkey2 = nrows + 17 # Write a few records. - cursor = self.session.open_cursor(uri) + cursor = ds.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows + 1): cursor[i] = i @@ -186,7 +187,7 @@ class test_flcs01(wttest.WiredTigerTestCase): self.session.rollback_transaction() # Evict the page to force reconciliation. - self.evict(uri, 1, 1) + self.evict(ds, uri, 1, 1) # The committed zeros should still be there. self.session.begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_flcs05.py b/src/third_party/wiredtiger/test/suite/test_flcs05.py index 8e8c116c892..cff43208d7d 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_flcs05.py +++ b/src/third_party/wiredtiger/test/suite/test_flcs05.py @@ -47,8 +47,8 @@ class test_flcs05(wttest.WiredTigerTestCase): self.assertEqual(cursor.get_value(), v) # Evict the page to force reconciliation. - def evict(self, uri, key, check_value): - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + def evict(self, ds, key, check_value): + evict_cursor = ds.open_cursor(ds.uri, None, "debug=(release_evict)") self.session.begin_transaction() v = evict_cursor[key] self.assertEqual(v, check_value) @@ -56,6 +56,7 @@ class test_flcs05(wttest.WiredTigerTestCase): self.session.rollback_transaction() evict_cursor.close() + @wttest.skip_for_hook("tiered", "fails at begin_transaction") # FIXME-WT-9809 def test_flcs(self): uri = "table:test_flcs05" nrows = 44 @@ -67,7 +68,7 @@ class test_flcs05(wttest.WiredTigerTestCase): updatekey2 = 37 appendkey1 = nrows + 10 - cursor = self.session.open_cursor(uri, None, 'overwrite=false') + cursor = ds.open_cursor(uri, None, 'overwrite=false') # Write a few records. #self.session.begin_transaction() @@ -94,7 +95,7 @@ class test_flcs05(wttest.WiredTigerTestCase): self.tryread(cursor, updatekey2, 0) # Deleted value that has been reconciled. - self.evict(ds.uri, updatekey2, 0) + self.evict(ds, updatekey2, 0) self.tryread(cursor, updatekey2, 0) # Deleted value in the append list. diff --git a/src/third_party/wiredtiger/test/suite/test_log04.py b/src/third_party/wiredtiger/test/suite/test_log04.py index 19a8d73bf68..7700aaad0fb 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_log04.py +++ b/src/third_party/wiredtiger/test/suite/test_log04.py @@ -51,6 +51,7 @@ class test_log04(wttest.WiredTigerTestCase): self.assertEqual(cursor[key], value) self.session.rollback_transaction() + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_logts(self): # Create logged and non-logged objects. The non-logged objects are in two versions, one is # updated with a commit timestamp and one is not. Update the logged and non-logged timestamp diff --git a/src/third_party/wiredtiger/test/suite/test_overwrite.py b/src/third_party/wiredtiger/test/suite/test_overwrite.py index e9aa353440a..55c04a407c3 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_overwrite.py +++ b/src/third_party/wiredtiger/test/suite/test_overwrite.py @@ -62,7 +62,7 @@ class test_overwrite(wttest.WiredTigerTestCase): ds.populate() # Insert of an existing record with overwrite off fails. - cursor = self.session.open_cursor(uri, None, "overwrite=false") + cursor = ds.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) cursor.set_value(ds.value(1000)) self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) @@ -71,26 +71,26 @@ class test_overwrite(wttest.WiredTigerTestCase): # configured and then the insert should succeed. This test is only for the insert method # because the update method's failure modes are for non-existent records, and you cannot # duplicate a cursor pointing to non-existent records. - cursor = self.session.open_cursor(uri, None, "overwrite=false") + cursor = ds.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") dupc.set_value(ds.value(1001)) self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) cursor.set_key(ds.key(6)) cursor.set_value(ds.value(1002)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. - cursor = self.session.open_cursor(uri, None, "overwrite=false") + cursor = ds.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) cursor.set_value(ds.value(1003)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) cursor.set_key(ds.key(201)) cursor.set_value(ds.value(1004)) self.assertEquals(cursor.insert(), 0) @@ -102,22 +102,22 @@ class test_overwrite(wttest.WiredTigerTestCase): ds.populate() # Remove of an existing record with overwrite off succeeds. - cursor = self.session.open_cursor(uri, None, "overwrite=false") + cursor = ds.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) self.assertEquals(cursor.remove(), 0) # Remove of an existing record with overwrite on succeeds. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) cursor.set_key(ds.key(6)) self.assertEquals(cursor.remove(), 0) # Remove of a non-existent record with overwrite off fails. - cursor = self.session.open_cursor(uri, None, "overwrite=false") + cursor = ds.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND) # Remove of a non-existent record with overwrite on fails. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) cursor.set_key(ds.key(201)) self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND) @@ -127,25 +127,25 @@ class test_overwrite(wttest.WiredTigerTestCase): ds.populate() # Update of an existing record with overwrite off succeeds. - cursor = self.session.open_cursor(uri, None, "overwrite=false") + cursor = ds.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) cursor.set_value(ds.value(1005)) self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) cursor.set_key(ds.key(6)) cursor.set_value(ds.value(1006)) self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. - cursor = self.session.open_cursor(uri, None, "overwrite=false") + cursor = ds.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) cursor.set_value(ds.value(1007)) self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) cursor.set_key(ds.key(201)) cursor.set_value(ds.value(1008)) self.assertEquals(cursor.update(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py index e748bc1c467..a08de6c3b09 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py @@ -63,6 +63,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): # between cursor : with timestamp between prepare and commit timestamps. # after cursor : with timestamp after commit timestamp. # Cursor with out read timestamp behaviour should be same after cursor behavior. + # + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_cursor_navigate_prepare_transaction(self): # Build an object. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py index 430032cc344..f7178b84c3a 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py @@ -122,6 +122,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # aborted self.check(uri, ds, nrows, nsessions, nkeys, 3, bigvalue1, bigvalue2) + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_prepare_hs(self): # Create a small table. uri = "table:test_prepare_hs01" diff --git a/src/third_party/wiredtiger/test/suite/test_reserve.py b/src/third_party/wiredtiger/test/suite/test_reserve.py index cc385402961..b33a363cb76 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_reserve.py +++ b/src/third_party/wiredtiger/test/suite/test_reserve.py @@ -69,7 +69,7 @@ class test_reserve(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 500, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor(uri, None, session=s) # Repeatedly update a record. for i in range(1, 5): @@ -121,7 +121,7 @@ class test_reserve(wttest.WiredTigerTestCase): # transaction (which should fail), repeatedly update a record and # commit. s2 = self.conn.open_session() - c2 = s2.open_cursor(uri, None) + c2 = ds.open_cursor(uri, None, session=s2) for i in range(1, 2): s.begin_transaction() c.set_key(ds.key(100)) @@ -146,7 +146,7 @@ class test_reserve(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 10, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor(uri, None, session=s) s.begin_transaction() msg = "/requires key be set/" self.assertRaisesWithMessage( @@ -160,7 +160,7 @@ class test_reserve(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 10, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor(uri, None, session=s) c.set_key(ds.key(5)) msg = "/only permitted in a running transaction/" self.assertRaisesWithMessage( @@ -174,7 +174,7 @@ class test_reserve(wttest.WiredTigerTestCase): ds = self.ds(self, uri, 10, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() - c = s.open_cursor(uri, None) + c = ds.open_cursor(uri, None, session=s) s.begin_transaction() c.set_key(ds.key(5)) self.assertEquals(c.reserve(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py index d3f8396b860..ca51b63c7c2 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py @@ -25,6 +25,7 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +import wttest from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base from wiredtiger import stat @@ -53,6 +54,7 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): config = 'cache_size=50MB,statistics=(all)' return config + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_rollback_to_stable(self): nrows = 1000 @@ -108,6 +110,7 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] self.assertEqual(restored_tombstones, nrows) + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_rollback_to_stable_with_aborted_updates(self): nrows = 1000 @@ -183,6 +186,7 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] self.assertEqual(restored_tombstones, nrows) + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_rollback_to_stable_with_history_tombstone(self): nrows = 1000 @@ -252,6 +256,7 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] self.assertEqual(restored_tombstones, nrows) + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_rollback_to_stable_with_stable_remove(self): nrows = 1000 # Create a table. diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py index 13b68e1f2a6..ce6d294f65c 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_truncate01.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py @@ -52,15 +52,17 @@ class test_truncate_arguments(wttest.WiredTigerTestCase): # either cursor specified, expect errors. def test_truncate_bad_args(self): uri = self.type + self.name - SimpleDataSet(self, uri, 100).populate() + ds = SimpleDataSet(self, uri, 100) + ds.populate() + msg = '/either a URI or start/stop cursors/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(None, None, None, None), msg) - cursor = self.session.open_cursor(uri, None) + lambda: ds.truncate(None, None, None, None), msg) + cursor = ds.open_cursor(uri, None) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(uri, cursor, None, None), msg) + lambda: ds.truncate(uri, cursor, None, None), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(uri, None, cursor, None), msg) + lambda: ds.truncate(uri, None, cursor, None), msg) # Test truncation of cursors where no key is set, expect errors. def test_truncate_cursor_notset(self): @@ -74,9 +76,9 @@ class test_truncate_arguments(wttest.WiredTigerTestCase): c2 = self.session.open_cursor(uri, None) c2.set_key(ds.key(10)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(None, c1, c2, None), msg) + lambda: ds.truncate(None, c1, c2, None), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(None, c2, c1, None), msg) + lambda: ds.truncate(None, c2, c1, None), msg) c1.close() c2.close() @@ -93,14 +95,17 @@ class test_truncate_uri(wttest.WiredTigerTestCase): uri = self.type + self.name # A simple, one-file file or table object. - SimpleDataSet(self, uri, 100).populate() - self.session.truncate(uri, None, None, None) + ds = SimpleDataSet(self, uri, 100) + ds.populate() + + ds.truncate(uri, None, None, None) confirm_empty(self, uri) self.dropUntilSuccess(self.session, uri) if self.type == "table:": - ComplexDataSet(self, uri, 100).populate() - self.session.truncate(uri, None, None, None) + cds = ComplexDataSet(self, uri, 100) + cds.populate() + cds.truncate(uri, None, None, None) confirm_empty(self, uri) self.dropUntilSuccess(self.session, uri) @@ -131,10 +136,10 @@ class test_truncate_cursor_order(wttest.WiredTigerTestCase): c2.set_key(ds.key(10)) msg = '/the start cursor position is after the stop cursor position/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(None, c1, c2, None), msg) + lambda: ds.truncate(None, c1, c2, None), msg) c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) - self.session.truncate(None, c1, c2, None) + ds.truncate(None, c1, c2, None) # Test truncation of cursors past the end of the object. class test_truncate_cursor_end(wttest.WiredTigerTestCase): @@ -162,7 +167,7 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): c1.set_key(ds.key(1000)) c2 = self.session.open_cursor(uri, None) c2.set_key(ds.key(2000)) - self.session.truncate(None, c1, c2, None) + ds.truncate(None, c1, c2, None) self.assertEqual(c1.close(), 0) self.assertEqual(c2.close(), 0) self.dropUntilSuccess(self.session, uri) @@ -174,7 +179,7 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): c1.set_key(ds.key(1000)) c2 = self.session.open_cursor(uri, None) c2.set_key(ds.key(2000)) - self.session.truncate(None, c1, c2, None) + ds.truncate(None, c1, c2, None) self.assertEqual(c1.close(), 0) self.assertEqual(c2.close(), 0) self.dropUntilSuccess(self.session, uri) @@ -223,6 +228,7 @@ class test_truncate_timestamp(wttest.WiredTigerTestCase): ]) # Test truncation without a timestamp, expect errors. + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_truncate_no_ts(self): uri = self.type + self.name msg = '/truncate operations may not yet be included/' @@ -232,9 +238,10 @@ class test_truncate_timestamp(wttest.WiredTigerTestCase): self.session.begin_transaction("no_timestamp=true") self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(uri, None, None, None), msg) + lambda: ds.truncate(uri, None, None, None), msg) # Test truncation of a logged object without a timestamp, expect success. + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_truncate_log_no_ts(self): uri = self.type + self.name @@ -242,7 +249,7 @@ class test_truncate_timestamp(wttest.WiredTigerTestCase): ds.populate() self.session.begin_transaction("no_timestamp=true") - self.session.truncate(uri, None, None, None) + ds.truncate(uri, None, None, None) # Test session.truncate. class test_truncate_cursor(wttest.WiredTigerTestCase): @@ -280,7 +287,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): def cursorKey(self, ds, uri, key): if key == -1: return None - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) cursor.set_key(ds.key(key)) return cursor @@ -289,7 +296,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): self.pr('truncateRangeAndCheck: ' + str(begin) + ',' + str(end)) cur1 = self.cursorKey(ds, uri, begin) cur2 = self.cursorKey(ds, uri, end) - self.session.truncate(None, cur1, cur2, None) + ds.truncate(None, cur1, cur2, None) if not cur1: begin = 1 else: @@ -305,7 +312,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): return # Check the expected values against the object. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) for i in range(begin, end + 1): expected[ds.key(i)] = [0] for k, v in expected.items(): @@ -507,7 +514,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # Build a dictionary of what the object should look like for # later comparison - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor() expected = {} for i in range(1, self.nentries + 1): expected[ds.key(i)] = ds.comparable_value(i) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate02.py b/src/third_party/wiredtiger/test/suite/test_truncate02.py index 234a941f8d7..ac0654a71f3 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_truncate02.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate02.py @@ -102,10 +102,10 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): self.assertEqual(count, expected) # Open a cursor in a new session and confirm how many records it sees. - def outside_count(self, isolation, expected): + def outside_count(self, ds, isolation, expected): s = self.conn.open_session() s.begin_transaction(isolation) - cursor = s.open_cursor(self.type + self.name, None) + cursor = ds.open_cursor(self.type + self.name, None, session=s) self.cursor_count(cursor, expected) s.close() @@ -130,7 +130,7 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): # Optionally add a few overflow records so we block fast delete on # those pages. if self.overflow: - cursor = self.session.open_cursor(uri, None, 'overwrite=false') + cursor = ds.open_cursor(uri, None, 'overwrite=false') for i in range(1, self.nentries, 3123): cursor.set_key(ds.key(i)) cursor.set_value(ds.value(i)) @@ -142,7 +142,7 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): # Optionally read/write a few rows before truncation. if self.readbefore or self.writebefore: - cursor = self.session.open_cursor(uri, None, 'overwrite=false') + cursor = ds.open_cursor(uri, None, 'overwrite=false') if self.readbefore: for i in range(1, self.nentries, 737): cursor.set_key(ds.key(i)) @@ -156,17 +156,17 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): # Begin a transaction, and truncate a big range of rows. self.session.begin_transaction(None) - start = self.session.open_cursor(uri, None) + start = ds.open_cursor(uri, None) start.set_key(ds.key(10)) - end = self.session.open_cursor(uri, None) + end = ds.open_cursor(uri, None) end.set_key(ds.key(self.nentries - 10)) - self.session.truncate(None, start, end, None) + ds.truncate(None, start, end, None) start.close() end.close() # Optionally read/write a few rows after truncation. if self.readafter or self.writeafter: - cursor = self.session.open_cursor(uri, None, 'overwrite=false') + cursor = ds.open_cursor(uri, None, 'overwrite=false') if self.readafter: for i in range(1, self.nentries, 1123): cursor.set_key(ds.key(i)) @@ -182,15 +182,15 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): # The number 19 comes from deleting row 10 (inclusive), to row N - 10, # exclusive, or 9 + 10 == 19. remaining = 19 - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) self.cursor_count(cursor, remaining) cursor.close() # A separate, read_committed cursor should not see the deleted records. - self.outside_count("isolation=read-committed", self.nentries) + self.outside_count(ds, "isolation=read-committed", self.nentries) # A separate, read_uncommitted cursor should see the deleted records. - self.outside_count("isolation=read-uncommitted", remaining) + self.outside_count(ds, "isolation=read-uncommitted", remaining) # Commit/rollback the transaction. if self.commit: @@ -199,7 +199,7 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): self.session.rollback_transaction() # Check a read_committed cursor sees the right records. - cursor = self.session.open_cursor(uri, None) + cursor = ds.open_cursor(uri, None) if self.commit: self.cursor_count(cursor, remaining) else: diff --git a/src/third_party/wiredtiger/test/suite/test_truncate03.py b/src/third_party/wiredtiger/test/suite/test_truncate03.py index 2cbcb21cee5..372436902a7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_truncate03.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate03.py @@ -73,11 +73,11 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # Truncate a big range of rows; the leaf pages aren't in memory, so # leaf page references will be deleted without being read. - start = self.session.open_cursor(self.uri, None) + start = ds.open_cursor(self.uri, None) start.set_key(ds.key(10)) - end = self.session.open_cursor(self.uri, None) + end = ds.open_cursor(self.uri, None) end.set_key(ds.key(self.nentries - 10)) - self.session.truncate(None, start, end, None) + ds.truncate(None, start, end, None) self.assertEqual(start.close(), 0) self.assertEqual(end.close(), 0) @@ -92,7 +92,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # mark pages with address-deleted cells dirty), then walk the tree so # we get a good look at all the internal pages and the address-deleted # cells. - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) cursor.set_key(ds.key(5)) cursor.set_value(changed_value) self.assertEqual(cursor.update(), 0) @@ -129,7 +129,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # creation of empty pages (once the underlying leaf page is freed, we # have to magic up a page if we need it). Confirm we can read/write # the value as well as write the page and get it back. - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) for i in range(3000, 7000, 137): k = ds.key(i) if self.value_format == '8t': @@ -152,7 +152,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): self.reopen_conn() self.session.verify(self.uri) - cursor = self.session.open_cursor(self.uri, None) + cursor = ds.open_cursor(self.uri, None) for i in range(3000, 7000, 137): k = ds.key(i) if self.value_format == '8t': diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index 0bc82571818..afda18043b6 100755 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -26,6 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +import wttimestamp class BaseDataSet(object): """ @@ -43,20 +44,33 @@ class BaseDataSet(object): self.config = kwargs.get('config', '') self.projection = kwargs.get('projection', '') + # If the timestamp generator is not set, get it from the test case. + self.timestamp = kwargs.get('timestamp', testcase.getTimestamp()) + def create(self): self.testcase.session.create(self.uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format + ',' + self.config) + def open_cursor(self, uri=None, todup=None, config=None, session=None): + if uri == None: + uri = self.uri + if session == None: + session = self.testcase.session + c = session.open_cursor(uri, None, config) + return wttimestamp.TimestampedCursor(c, self.timestamp, self.testcase) + + def truncate(self, uri, c1, c2, config, session=None): + if session == None: + session = self.testcase.session + with wttimestamp.session_timestamped_transaction(session, self.timestamp): + return session.truncate(uri, c1, c2, config) + def store_one_cursor(self, c, i): c[self.key(i)] = self.value(i) - def store_range_cursor(self, c, key, count): - for i in range(key, key + count): - self.store_one(c, i) - def store_range(self, key, count): - c = self.testcase.session.open_cursor(self.uri, None) + c = self.open_cursor() for i in range(key, key + count): self.store_one_cursor(c, i) c.close() @@ -130,8 +144,7 @@ class BaseDataSet(object): def check(self): self.testcase.pr('check: ' + self.uri) - cursor = self.testcase.session.open_cursor( - self.uri + self.projection, None, None) + cursor = self.open_cursor(self.uri + self.projection) self.check_cursor(cursor) cursor.close() @@ -198,7 +211,7 @@ class SimpleIndexDataSet(SimpleDataSet): BaseDataSet.check(self) # Check values in the index. - idxcursor = self.testcase.session.open_cursor(self.indexname) + idxcursor = self.open_cursor(self.indexname) for i in range(1, self.rows + 1): k = self.key(i) v = self.value(i) @@ -404,7 +417,7 @@ class ProjectionIndexDataSet(BaseDataSet): BaseDataSet.check(self) # Check values in the index. - idxcursor = self.testcase.session.open_cursor( + idxcursor = self.open_cursor( self.indexname + '(v1,k,v2,v0)') self.check_index_cursor(idxcursor) idxcursor.close() @@ -435,7 +448,7 @@ class TrackedComplexDataSet(ComplexDataSet): # override def store_one_cursor(self, c, i): self.track_values[i] = self.store_count(i) + 1 - c[self.key(i)] = self.value(i) + super().store_one_cursor(c, i) # Redefine the value stored to get bigger depending on the multiplier, # and to mix up the value depending on how many times it has been updated. @@ -490,7 +503,7 @@ class TrackedSimpleDataSet(SimpleDataSet): # override def store_one_cursor(self, c, i): self.track_values[i] = self.store_count(i) + 1 - c[self.key(i)] = self.value(i) + super().store_one_cursor(c, i) # Redefine the value stored to get bigger depending on the multiplier, # and to mix up the value depending on how many times it has been updated. diff --git a/src/third_party/wiredtiger/test/suite/wthooks.py b/src/third_party/wiredtiger/test/suite/wthooks.py index 3cdd2e32475..2ee0e26c499 100755 --- a/src/third_party/wiredtiger/test/suite/wthooks.py +++ b/src/third_party/wiredtiger/test/suite/wthooks.py @@ -246,6 +246,14 @@ class WiredTigerHookManager(object): def get_platform_api(self): return self.platform_api + # Returns a list of hook names that use something on the list + def hooks_using(self, use_list): + ret = [] + for hook in self.hooks: + if hook.uses(use_list): + ret.append(hook.name) + return ret + class HookCreatorProxy(object): def __init__(self, hookmgr, clazz): self.hookmgr = hookmgr @@ -284,8 +292,23 @@ class WiredTigerHookCreator(ABC): """Set up all hooks using add_*_hook methods.""" return + # default version of uses, can be overridden. If the hook uses or provides + # a capability on the list, it should return True. + def uses(self, use_list): + return False + class WiredTigerHookPlatformAPI(ABC): @abstractmethod + def setUp(self): + """Called at the beginning of a test case""" + pass + + @abstractmethod + def tearDown(self): + """Called at the termination of a test case""" + pass + + @abstractmethod def tableExists(self, name): """Return boolean if local files exist for the table with the given base name""" pass @@ -295,7 +318,18 @@ class WiredTigerHookPlatformAPI(ABC): """The first local backing file name created for this URI.""" pass + @abstractmethod + def getTimestamp(self): + """The timestamp generator for this test case.""" + pass + class DefaultPlatformAPI(WiredTigerHookPlatformAPI): + def setUp(self): + pass + + def tearDown(self): + pass + def tableExists(self, name): tablename = name + ".wt" return os.path.exists(tablename) @@ -307,3 +341,7 @@ class DefaultPlatformAPI(WiredTigerHookPlatformAPI): return uri[5:] else: raise Exception('bad uri') + + # By default, there is no automatic timestamping by test infrastructure classes. + def getTimestamp(self): + return None diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index dbe152e65df..221bd5e13b7 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -337,6 +337,10 @@ class WiredTigerTestCase(unittest.TestCase): def initialFileName(self, name): return self.platform_api.initialFileName(name) + # Return the WiredTigerTimestamp for this testcase, or None if there is none. + def getTimestamp(self): + return self.platform_api.getTimestamp() + def __str__(self): # when running with scenarios, if the number_scenarios() method # is used, then each scenario is given a number, which can @@ -570,6 +574,8 @@ class WiredTigerTestCase(unittest.TestCase): # tearDown needs connections list, set it here in case the open fails. self._connections = [] self._failed = None # set to True/False during teardown. + + self.platform_api.setUp() self.origcwd = os.getcwd() shutil.rmtree(self.testdir, ignore_errors=True) if os.path.exists(self.testdir): @@ -635,6 +641,8 @@ class WiredTigerTestCase(unittest.TestCase): self._failed = error or failure or exc_failure passed = not self._failed + self.platform_api.tearDown() + # Download the files from the S3 bucket for tiered tests if the test fails or preserve is # turned on. if hasattr(self, 'ss_name') and self.ss_name == 's3_store' and not self.skipped and \ @@ -1027,6 +1035,33 @@ def longtest(description): else: return runit_decorator +def prevent(what): + """ + Used as a function decorator, for example, @wttest.prevent("timestamp"). + The decorator indicates that this test function uses some facility (in the + example, it uses its own timestamps), and prevents hooks that try to manage + timestamps from running the test. + """ + def runit_decorator(func): + return func + hooks_using = WiredTigerTestCase._hookmgr.hooks_using(what) + if len(hooks_using) > 0: + return unittest.skip("function uses {}, which is incompatible with hooks: {}".format(what, hooks_using)) + else: + return runit_decorator + +def skip_for_hook(hookname, description): + """ + Used as a function decorator, for example, @wttest.skip_for_hook("tiered", "fails at commit_transaction"). + The decorator indicates that this test function fails with the hook, which should be investigated. + """ + def runit_decorator(func): + return func + if hookname in WiredTigerTestCase.hook_names: + return unittest.skip("because running with hook '{}': {}".format(hookname, description)) + else: + return runit_decorator + def islongtest(): return WiredTigerTestCase._longtest diff --git a/src/third_party/wiredtiger/test/suite/wttimestamp.py b/src/third_party/wiredtiger/test/suite/wttimestamp.py new file mode 100755 index 00000000000..26fff886d65 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/wttimestamp.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +from contextlib import contextmanager +import wttest, wiredtiger + +# Timestamp abstraction class +class WiredTigerTimeStamp(object): + def __init__(self, initial_timestamp=1): + self.ts = initial_timestamp + + def incr(self): + self.ts += 1 + + def get_incr(self): + ret = self.ts + self.ts += 1 + return ret + + def get(self): + return self.ts + +# This allows us to easily "wrap" an operation in a transaction at the +# next timestamp. This global function version can be called externally. +@contextmanager +def session_timestamped_transaction(session, timestamper): + need_commit = False + if timestamper != None and not getattr(session, "_has_transaction", False): + session.begin_transaction() + need_commit = True + yield + if need_commit: + config = 'commit_timestamp=%x' % timestamper.get_incr() + #wttest.WiredTigerTestCase.tty('commit_transaction ' + config) + session.commit_transaction(config) + elif timestamper != None: + config = 'commit_timestamp=%x' % timestamper.get_incr() + #wttest.WiredTigerTestCase.tty('timestamp_transaction ' + config) + session.timestamp_transaction(config) + +# This class acts as a "proxy" for a Cursor. All methods, etc. +# are passed to the implementation object (via __getattr__), +# except for the ones that we explicitly override here. +class TimestampedCursor(wiredtiger.Cursor): + def __init__(self, cursor, timeStamper, testcase): + self._cursor = cursor + self._timeStamper = timeStamper + self._testcase = testcase + + def __getattr__(self, name): + return getattr(self._cursor, name) + + # A more convenient way to "wrap" an operation in a transaction + @contextmanager + def timestamped_transaction(self): + # Prefer the _session object if available, it returns a Python + # Session object that is 1-1 mapped to the WT_SESSION in the C API. + session = getattr(self._cursor, "_session", self._cursor.session) + timestamper = self._timeStamper + with session_timestamped_transaction(session, timestamper): + yield + + # Overrides Cursor.insert + def insert(self): + with self.timestamped_transaction(): + return self._cursor.insert() + + # Overrides Cursor.update + def update(self): + with self.timestamped_transaction(): + return self._cursor.update() + + # Overrides Cursor.remove + def remove(self): + with self.timestamped_transaction(): + return self._cursor.remove() + + # Overrides Cursor.modify + def modify(self, mods): + with self.timestamped_transaction(): + return self._cursor.modify(mods) |