diff options
Diffstat (limited to 'src/third_party/wiredtiger')
15 files changed, 46 insertions, 101 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 61933b6c6f0..9c181f070e5 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "0dc273cfce453cdf518ae5a61bc54bf996074eb6" + "commit": "8587a3713ed1f9b6372a135d433ac8c3957d4f42" } diff --git a/src/third_party/wiredtiger/src/schema/schema_truncate.c b/src/third_party/wiredtiger/src/schema/schema_truncate.c index be9a62a56ba..3b1cfcfd1e2 100644 --- a/src/third_party/wiredtiger/src/schema/schema_truncate.c +++ b/src/third_party/wiredtiger/src/schema/schema_truncate.c @@ -41,16 +41,23 @@ err: * Truncate for a tiered data source. */ static int -__truncate_tiered(WT_SESSION_IMPL *session, const char *uri) +__truncate_tiered(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) { WT_DECL_RET; + WT_TIERED *tiered; + u_int i; WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, WT_DHANDLE_EXCLUSIVE)); + tiered = (WT_TIERED *)session->dhandle; WT_STAT_DATA_INCR(session, cursor_truncate); - WT_WITHOUT_DHANDLE(session, ret = __wt_session_range_truncate(session, uri, NULL, NULL)); - WT_ERR(ret); + /* Truncate the tiered entries. */ + for (i = 0; i < WT_TIERED_MAX_TIERS; i++) { + if (tiered->tiers[i].tier == NULL) + continue; + WT_ERR(__wt_schema_truncate(session, tiered->tiers[i].name, cfg)); + } err: WT_TRET(__wt_session_release_dhandle(session)); @@ -105,7 +112,7 @@ __wt_schema_truncate(WT_SESSION_IMPL *session, const char *uri, const char *cfg[ else if (WT_PREFIX_SKIP(tablename, "table:")) ret = __truncate_table(session, tablename, cfg); else if (WT_PREFIX_MATCH(uri, "tiered:")) - ret = __truncate_tiered(session, uri); + ret = __truncate_tiered(session, uri, cfg); else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) ret = dsrc->truncate == NULL ? __truncate_dsrc(session, uri) : diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py index 06df8a328b9..835800ec5b2 100755 --- a/src/third_party/wiredtiger/test/suite/hook_tiered.py +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -96,15 +96,15 @@ def wiredtiger_open_tiered(ignored_self, args): # however, we alter several other API methods that would do weird things with # a different tiered_storage configuration. So better to skip the test entirely. if 'tiered_storage=' in curconfig: - skipTest("cannot run tiered hook on a test that already uses tiered storage") + testcase.skipTest("cannot run tiered hook on a test that already uses tiered storage") # Similarly if this test is already set up to run tiered vs non-tiered scenario, let's # not get in the way. if hasattr(testcase, 'tiered_conn_config'): - skipTest("cannot run tiered hook on a test that already includes TieredConfigMixin") + testcase.skipTest("cannot run tiered hook on a test that already includes TieredConfigMixin") if 'in_memory=true' in curconfig: - skipTest("cannot run tiered hook on a test that is in-memory") + testcase.skipTest("cannot run tiered hook on a test that is in-memory") # Mark this test as readonly, but don't disallow it. See testcase_is_readonly(). if 'readonly=true' in curconfig: @@ -159,10 +159,6 @@ def testcase_has_failed(): testcase = WiredTigerTestCase.currentTestCase() return testcase.failed() -def skipTest(comment): - testcase = WiredTigerTestCase.currentTestCase() - testcase.skipTest(comment) - # Called to replace Connection.close # Insert a call to flush_tier before closing connection. def connection_close_replace(orig_connection_close, connection_self, config): @@ -179,16 +175,9 @@ def connection_close_replace(orig_connection_close, connection_self, config): return ret # Called to replace Session.checkpoint. -# We add a call to flush_tier during every checkpoint to make sure we are exercising tiered +# We add a call to flush_tier after the checkpoint to make sure we are exercising tiered # functionality. def session_checkpoint_replace(orig_session_checkpoint, session_self, config): - # FIXME-WT-10771 We cannot do named checkpoints with tiered storage objects. - # We can't really continue the test without the name, as the name will certainly be used. - testcase = WiredTigerTestCase.currentTestCase() - if config == None: - config = '' - if 'name=' in config: - skipTest('named checkpoints do not work in tiered storage') # We cannot call flush_tier on a readonly connection. if not testcase_is_readonly(): config += ',flush_tier=(enabled,force=true)' @@ -230,9 +219,11 @@ def session_create_replace(orig_session_create, session_self, uri, config): # do statistics on (tiered) table data sources, as that is not yet supported. def session_open_cursor_replace(orig_session_open_cursor, session_self, uri, dupcursor, config): if uri != None and (uri.startswith("statistics:table:") or uri.startswith("statistics:file:")): - skipTest("statistics on tiered tables not yet implemented") + testcase = WiredTigerTestCase.currentTestCase() + testcase.skipTest("statistics on tiered tables not yet implemented") if uri != None and uri.startswith("backup:"): - skipTest("backup on tiered tables not yet implemented") + testcase = WiredTigerTestCase.currentTestCase() + testcase.skipTest("backup on tiered tables not yet implemented") return orig_session_open_cursor(session_self, uri, dupcursor, config) # Called to replace Session.rename @@ -278,16 +269,6 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): # Override some platform APIs self.platform_api = TieredPlatformAPI(arg) - # Our current tiered storage implementation has a slow version of truncate, and - # some tests are sensitive to that. - # - # FIXME-WT-11023: when we implement a fast truncate for tiered storage, we might remove - # this, and visit tests that are marked with @wttest.prevent(..."slow_truncate"...) - def uses(self, use_list): - if "slow_truncate" in use_list: - return True - return False - # Is this test one we should skip? def skip_test(self, test): # Skip any test that contains one of these strings as a substring @@ -355,6 +336,19 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): "test_rollback_to_stable36.test_rollback_to_stable", "test_sweep03.test_disable_idle_timeout_drop", "test_sweep03.test_disable_idle_timeout_drop_force", + "test_truncate01.test_truncate_cursor", + "test_truncate01.test_truncate_cursor_end", + "test_truncate01.test_truncate_timestamp", + "test_truncate01.test_truncate_uri", + "test_truncate10.test_truncate10", + "test_truncate12.test_truncate12", + "test_truncate13.test_truncate", + "test_truncate14.test_truncate", + "test_truncate16.test_truncate16", + "test_truncate18.test_truncate18", + "test_truncate15.test_truncate15", + "test_truncate19.test_truncate19", + "test_truncate20.test_truncate20", "test_txn22.test_corrupt_meta", "test_verbose01.test_verbose_single", "test_verbose02.test_verbose_single", @@ -380,10 +374,6 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): self.Connection['close'] = (wthooks.HOOK_REPLACE, lambda s, config=None: connection_close_replace(orig_connection_close, s, config)) - orig_session_checkpoint = self.Session['checkpoint'] - self.Session['checkpoint'] = (wthooks.HOOK_REPLACE, lambda s, config=None: - session_checkpoint_replace(orig_session_checkpoint, s, config)) - orig_session_compact = self.Session['compact'] self.Session['compact'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_compact_replace(orig_session_compact, s, uri, config)) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py index 47e94edf6a9..f2c13e536e8 100755 --- a/src/third_party/wiredtiger/test/suite/test_truncate01.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py @@ -227,10 +227,8 @@ class test_truncate_timestamp(wttest.WiredTigerTestCase): ('table', dict(type='table:')) ]) - # The next two functions assume that truncate maps to a fast truncate, which requires timestamps. - # Prevent these from running under a hook that is also adding timestamps to transactions. - # Also prevent these from running under a hook that will cause truncate to use a slow path. - @wttest.prevent(["timestamp", "slow_truncate"]) # prevent the use of hooks that manage timestamps + # Test truncation without a timestamp, expect errors. + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_truncate_no_ts(self): # Truncate with no timestamps is not allowed only in standalone builds. if not wiredtiger.standalone_build(): @@ -247,7 +245,7 @@ class test_truncate_timestamp(wttest.WiredTigerTestCase): lambda: ds.truncate(uri, None, None, None), msg) # Test truncation of a logged object without a timestamp, expect success. - @wttest.prevent(["timestamp", "slow_truncate"]) # prevent the use of hooks that manage timestamps + @wttest.prevent(["timestamp"]) # prevent the use of hooks that manage timestamps def test_truncate_log_no_ts(self): uri = self.type + self.name @@ -428,7 +426,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # Insert the records that aren't skipped or inserted. start = begin_skipped + begin_insert stop = self.nentries - (end_skipped + end_insert) - cursor = ds.open_cursor(uri, None) + cursor = self.session.open_cursor(uri, None) for i in range(start + 1, stop + 1): k = ds.key(i) v = ds.value(i) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate10.py b/src/third_party/wiredtiger/test/suite/test_truncate10.py index 91ccbd95a5b..8fc7e1f6a6f 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate10.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate10.py @@ -162,11 +162,7 @@ class test_truncate10(wttest.WiredTigerTestCase): # or running on FLCS where it isn't supported.) stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t' or self.trunc_with_remove: + if self.value_format == '8t' or self.trunc_with_remove: self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate11.py b/src/third_party/wiredtiger/test/suite/test_truncate11.py index cd0d0936ca2..10ff07cd2ee 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate11.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate11.py @@ -47,7 +47,6 @@ class test_truncate11(wttest.WiredTigerTestCase): scenarios = make_scenarios(format_values) - @wttest.skip_for_hook("tiered", "test depends on regular checkpoints running") def test_truncate11(self): # Create a large table with lots of pages. uri = "table:test_truncate11" diff --git a/src/third_party/wiredtiger/test/suite/test_truncate12.py b/src/third_party/wiredtiger/test/suite/test_truncate12.py index e9b3cedec78..0a2102f200d 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate12.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate12.py @@ -174,11 +174,7 @@ class test_truncate12(wttest.WiredTigerTestCase): # or running on FLCS where it isn't supported.) stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t' or self.trunc_with_remove: + if self.value_format == '8t' or self.trunc_with_remove: self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate13.py b/src/third_party/wiredtiger/test/suite/test_truncate13.py index e61eb314d35..e17e750af20 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate13.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate13.py @@ -223,11 +223,7 @@ class test_truncate13(wttest.WiredTigerTestCase): # (Or if we are running with trunc_with_remove.) stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t' or self.trunc_with_remove: + if self.value_format == '8t' or self.trunc_with_remove: self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate14.py b/src/third_party/wiredtiger/test/suite/test_truncate14.py index 3df9a669371..5b037718ced 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate14.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate14.py @@ -213,11 +213,7 @@ class test_truncate14(wttest.WiredTigerTestCase): # (Except if we're running with trunc_with_remove.) stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.trunc_with_remove: + if self.trunc_with_remove: self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate15.py b/src/third_party/wiredtiger/test/suite/test_truncate15.py index 2b34f14f113..f9b8c5c62e2 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate15.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate15.py @@ -163,11 +163,7 @@ class test_truncate15(wttest.WiredTigerTestCase): # support, so assert we didn't. stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t': + if self.value_format == '8t': self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate16.py b/src/third_party/wiredtiger/test/suite/test_truncate16.py index 0a5ecdc16eb..0b192bf26cf 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate16.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate16.py @@ -145,11 +145,7 @@ class test_truncate16(wttest.WiredTigerTestCase): # or running on FLCS where it isn't supported.) stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t' or self.trunc_with_remove: + if self.value_format == '8t' or self.trunc_with_remove: self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) @@ -172,11 +168,7 @@ class test_truncate16(wttest.WiredTigerTestCase): # (But not if we weren't fast-deleting.) stat_cursor = self.session.open_cursor('statistics:', None, None) read_deleted = stat_cursor[stat.conn.cache_read_deleted][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t' or self.trunc_with_remove: + if self.value_format == '8t' or self.trunc_with_remove: self.assertEqual(read_deleted, 0) else: self.assertEqual(read_deleted, 1) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate17.py b/src/third_party/wiredtiger/test/suite/test_truncate17.py index d47e94151dd..8e15b4c5de8 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate17.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate17.py @@ -168,11 +168,7 @@ class test_truncate17(wttest.WiredTigerTestCase): # or running on FLCS where it isn't supported.) stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t' or self.trunc_with_remove: + if self.value_format == '8t' or self.trunc_with_remove: self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate18.py b/src/third_party/wiredtiger/test/suite/test_truncate18.py index 6e60aef0e8b..cc2ff934fb2 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate18.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate18.py @@ -169,11 +169,7 @@ class test_truncate18(wttest.WiredTigerTestCase): # or running on FLCS where it isn't supported.) stat_cursor = self.session.open_cursor('statistics:', None, None) fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2] - if self.runningHook('tiered'): - # There's no way the test can guess whether fast delete is possible when - # flush_tier calls are "randomly" inserted. - pass - elif self.value_format == '8t' or self.trunc_with_remove: + if self.value_format == '8t' or self.trunc_with_remove: self.assertEqual(fastdelete_pages, 0) else: self.assertGreater(fastdelete_pages, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate19.py b/src/third_party/wiredtiger/test/suite/test_truncate19.py index b2bf1aef340..108ef7792c0 100755..100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate19.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate19.py @@ -56,7 +56,6 @@ class test_truncate19(wttest.WiredTigerTestCase): self.session.truncate(None, None, hicursor, None) self.session.commit_transaction() - @wttest.skip_for_hook("tiered", "test depends of sizes of associated file objects") def test_truncate19(self): uri = 'table:oplog' nrows = 1000000 diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 174d10a8283..2419848e3c4 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -348,8 +348,6 @@ class WiredTigerTestCase(unittest.TestCase): if seedw != 0 and seedz != 0: WiredTigerTestCase._randomseed = True WiredTigerTestCase._seeds = [seedw, seedz] - # We don't have a lot of output, but we want to see it right away. - sys.stdout.reconfigure(line_buffering=True) WiredTigerTestCase._globalSetup = True @staticmethod @@ -678,8 +676,6 @@ class WiredTigerTestCase(unittest.TestCase): os.chdir(self.testdir) with open('testname.txt', 'w+') as namefile: namefile.write(str(self) + '\n') - if WiredTigerTestCase._verbose >= 2: - print("[pid:{}]: {}: starting".format(os.getpid(), str(self))) self.fdSetUp() self._threadLocal.currentTestCase = self self.ignoreTearDownLogs = False @@ -834,9 +830,6 @@ class WiredTigerTestCase(unittest.TestCase): self.assertEqual(ret, wiredtiger.WT_NOTFOUND) bkp_cursor.close() - def runningHook(self, name): - return name in WiredTigerTestCase.hook_names - # Set a Python breakpoint. When this function is called, # the python debugger will be called as described here: # https://docs.python.org/3/library/pdb.html @@ -988,11 +981,6 @@ class WiredTigerTestCase(unittest.TestCase): return '%x' % t def dropUntilSuccess(self, session, uri, config=None): - # Most test cases consider a drop, and especially a 'drop until success', - # to completely remove a file's artifacts, so that the name can be reused. - # For tiered storage, this means removing associated cloud objects. - if self.runningHook('tiered') and config == None: - config = 'force=true,remove_shared=true' while True: try: session.drop(uri, config) |