Import wiredtiger: 8587a3713ed1f9b6372a135d433ac8c3957d4f42 from branch mongodb-master

ref: 0dc273cfce..8587a3713e
for: 7.1.0-rc0

Revert "WT-10971 Fix truncate for tiered storage."

15 files changed, 46 insertions, 101 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 61933b6c6f0..9c181f070e5 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "0dc273cfce453cdf518ae5a61bc54bf996074eb6"
+    "commit": "8587a3713ed1f9b6372a135d433ac8c3957d4f42"
 }
diff --git a/src/third_party/wiredtiger/src/schema/schema_truncate.c b/src/third_party/wiredtiger/src/schema/schema_truncate.c
index be9a62a56ba..3b1cfcfd1e2 100644
--- a/src/third_party/wiredtiger/src/schema/schema_truncate.c
+++ b/src/third_party/wiredtiger/src/schema/schema_truncate.c
@@ -41,16 +41,23 @@ err:
  *     Truncate for a tiered data source.
  */
 static int
-__truncate_tiered(WT_SESSION_IMPL *session, const char *uri)
+__truncate_tiered(WT_SESSION_IMPL *session, const char *uri, const char *cfg[])
 {
     WT_DECL_RET;
+    WT_TIERED *tiered;
+    u_int i;
 
     WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, WT_DHANDLE_EXCLUSIVE));
+    tiered = (WT_TIERED *)session->dhandle;
 
     WT_STAT_DATA_INCR(session, cursor_truncate);
 
-    WT_WITHOUT_DHANDLE(session, ret = __wt_session_range_truncate(session, uri, NULL, NULL));
-    WT_ERR(ret);
+    /* Truncate the tiered entries. */
+    for (i = 0; i < WT_TIERED_MAX_TIERS; i++) {
+        if (tiered->tiers[i].tier == NULL)
+            continue;
+        WT_ERR(__wt_schema_truncate(session, tiered->tiers[i].name, cfg));
+    }
 
 err:
     WT_TRET(__wt_session_release_dhandle(session));
@@ -105,7 +112,7 @@ __wt_schema_truncate(WT_SESSION_IMPL *session, const char *uri, const char *cfg[
     else if (WT_PREFIX_SKIP(tablename, "table:"))
         ret = __truncate_table(session, tablename, cfg);
     else if (WT_PREFIX_MATCH(uri, "tiered:"))
-        ret = __truncate_tiered(session, uri);
+        ret = __truncate_tiered(session, uri, cfg);
     else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL)
         ret = dsrc->truncate == NULL ?
           __truncate_dsrc(session, uri) :
diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py
index 06df8a328b9..835800ec5b2 100755
--- a/src/third_party/wiredtiger/test/suite/hook_tiered.py
+++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py
@@ -96,15 +96,15 @@ def wiredtiger_open_tiered(ignored_self, args):
     # however, we alter several other API methods that would do weird things with
     # a different tiered_storage configuration. So better to skip the test entirely.
     if 'tiered_storage=' in curconfig:
-        skipTest("cannot run tiered hook on a test that already uses tiered storage")
+        testcase.skipTest("cannot run tiered hook on a test that already uses tiered storage")
 
     # Similarly if this test is already set up to run tiered vs non-tiered scenario, let's
     # not get in the way.
     if hasattr(testcase, 'tiered_conn_config'):
-        skipTest("cannot run tiered hook on a test that already includes TieredConfigMixin")
+        testcase.skipTest("cannot run tiered hook on a test that already includes TieredConfigMixin")
 
     if 'in_memory=true' in curconfig:
-        skipTest("cannot run tiered hook on a test that is in-memory")
+        testcase.skipTest("cannot run tiered hook on a test that is in-memory")
 
     # Mark this test as readonly, but don't disallow it.  See testcase_is_readonly().
     if 'readonly=true' in curconfig:
@@ -159,10 +159,6 @@ def testcase_has_failed():
     testcase = WiredTigerTestCase.currentTestCase()
     return testcase.failed()
 
-def skipTest(comment):
-    testcase = WiredTigerTestCase.currentTestCase()
-    testcase.skipTest(comment)
-
 # Called to replace Connection.close
 # Insert a call to flush_tier before closing connection.
 def connection_close_replace(orig_connection_close, connection_self, config):
@@ -179,16 +175,9 @@ def connection_close_replace(orig_connection_close, connection_self, config):
     return ret
 
 # Called to replace Session.checkpoint.
-# We add a call to flush_tier during every checkpoint to make sure we are exercising tiered
+# We add a call to flush_tier after the checkpoint to make sure we are exercising tiered
 # functionality.
 def session_checkpoint_replace(orig_session_checkpoint, session_self, config):
-    # FIXME-WT-10771 We cannot do named checkpoints with tiered storage objects.
-    # We can't really continue the test without the name, as the name will certainly be used.
-    testcase = WiredTigerTestCase.currentTestCase()
-    if config == None:
-        config = ''
-    if 'name=' in config:
-        skipTest('named checkpoints do not work in tiered storage')
     # We cannot call flush_tier on a readonly connection.
     if not testcase_is_readonly():
         config += ',flush_tier=(enabled,force=true)'
@@ -230,9 +219,11 @@ def session_create_replace(orig_session_create, session_self, uri, config):
 # do statistics on (tiered) table data sources, as that is not yet supported.
 def session_open_cursor_replace(orig_session_open_cursor, session_self, uri, dupcursor, config):
     if uri != None and (uri.startswith("statistics:table:") or uri.startswith("statistics:file:")):
-        skipTest("statistics on tiered tables not yet implemented")
+        testcase = WiredTigerTestCase.currentTestCase()
+        testcase.skipTest("statistics on tiered tables not yet implemented")
     if uri != None and uri.startswith("backup:"):
-        skipTest("backup on tiered tables not yet implemented")
+        testcase = WiredTigerTestCase.currentTestCase()
+        testcase.skipTest("backup on tiered tables not yet implemented")
     return orig_session_open_cursor(session_self, uri, dupcursor, config)
 
 # Called to replace Session.rename
@@ -278,16 +269,6 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator):
         # Override some platform APIs
         self.platform_api = TieredPlatformAPI(arg)
 
-    # Our current tiered storage implementation has a slow version of truncate, and
-    # some tests are sensitive to that.
-    #
-    # FIXME-WT-11023: when we implement a fast truncate for tiered storage, we might remove
-    # this, and visit tests that are marked with @wttest.prevent(..."slow_truncate"...)
-    def uses(self, use_list):
-        if "slow_truncate" in use_list:
-            return True
-        return False
-
     # Is this test one we should skip?
     def skip_test(self, test):
         # Skip any test that contains one of these strings as a substring
@@ -355,6 +336,19 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator):
                 "test_rollback_to_stable36.test_rollback_to_stable",
                 "test_sweep03.test_disable_idle_timeout_drop",
                 "test_sweep03.test_disable_idle_timeout_drop_force",
+                "test_truncate01.test_truncate_cursor",
+                "test_truncate01.test_truncate_cursor_end",
+                "test_truncate01.test_truncate_timestamp",
+                "test_truncate01.test_truncate_uri",
+                "test_truncate10.test_truncate10",
+                "test_truncate12.test_truncate12",
+                "test_truncate13.test_truncate",
+                "test_truncate14.test_truncate",
+                "test_truncate16.test_truncate16",
+                "test_truncate18.test_truncate18",
+                "test_truncate15.test_truncate15",
+                "test_truncate19.test_truncate19",
+                "test_truncate20.test_truncate20",
                 "test_txn22.test_corrupt_meta",
                 "test_verbose01.test_verbose_single",
                 "test_verbose02.test_verbose_single",
@@ -380,10 +374,6 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator):
         self.Connection['close'] = (wthooks.HOOK_REPLACE, lambda s, config=None:
           connection_close_replace(orig_connection_close, s, config))
 
-        orig_session_checkpoint = self.Session['checkpoint']
-        self.Session['checkpoint'] =  (wthooks.HOOK_REPLACE, lambda s, config=None:
-          session_checkpoint_replace(orig_session_checkpoint, s, config))
-
         orig_session_compact = self.Session['compact']
         self.Session['compact'] =  (wthooks.HOOK_REPLACE, lambda s, uri, config=None:
           session_compact_replace(orig_session_compact, s, uri, config))
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py
index 47e94edf6a9..f2c13e536e8 100755
--- a/src/third_party/wiredtiger/test/suite/test_truncate01.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py
@@ -227,10 +227,8 @@ class test_truncate_timestamp(wttest.WiredTigerTestCase):
         ('table', dict(type='table:'))
     ])
 
-    # The next two functions assume that truncate maps to a fast truncate, which requires timestamps.
-    # Prevent these from running under a hook that is also adding timestamps to transactions.
-    # Also prevent these from running under a hook that will cause truncate to use a slow path.
-    @wttest.prevent(["timestamp", "slow_truncate"])  # prevent the use of hooks that manage timestamps
+    # Test truncation without a timestamp, expect errors.
+    @wttest.prevent(["timestamp"])  # prevent the use of hooks that manage timestamps
     def test_truncate_no_ts(self):
         # Truncate with no timestamps is not allowed only in standalone builds.
         if not wiredtiger.standalone_build():
@@ -247,7 +245,7 @@ class test_truncate_timestamp(wttest.WiredTigerTestCase):
             lambda: ds.truncate(uri, None, None, None), msg)
 
     # Test truncation of a logged object without a timestamp, expect success.
-    @wttest.prevent(["timestamp", "slow_truncate"])  # prevent the use of hooks that manage timestamps
+    @wttest.prevent(["timestamp"])  # prevent the use of hooks that manage timestamps
     def test_truncate_log_no_ts(self):
         uri = self.type + self.name
 
@@ -428,7 +426,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
                 # Insert the records that aren't skipped or inserted.
                 start = begin_skipped + begin_insert
                 stop = self.nentries - (end_skipped + end_insert)
-                cursor = ds.open_cursor(uri, None)
+                cursor = self.session.open_cursor(uri, None)
                 for i in range(start + 1, stop + 1):
                     k = ds.key(i)
                     v = ds.value(i)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate10.py b/src/third_party/wiredtiger/test/suite/test_truncate10.py
index 91ccbd95a5b..8fc7e1f6a6f 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate10.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate10.py
@@ -162,11 +162,7 @@ class test_truncate10(wttest.WiredTigerTestCase):
         # or running on FLCS where it isn't supported.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t' or self.trunc_with_remove:
+        if self.value_format == '8t' or self.trunc_with_remove:
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate11.py b/src/third_party/wiredtiger/test/suite/test_truncate11.py
index cd0d0936ca2..10ff07cd2ee 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate11.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate11.py
@@ -47,7 +47,6 @@ class test_truncate11(wttest.WiredTigerTestCase):
 
     scenarios = make_scenarios(format_values)
 
-    @wttest.skip_for_hook("tiered", "test depends on regular checkpoints running")
     def test_truncate11(self):
         # Create a large table with lots of pages.
         uri = "table:test_truncate11"
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate12.py b/src/third_party/wiredtiger/test/suite/test_truncate12.py
index e9b3cedec78..0a2102f200d 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate12.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate12.py
@@ -174,11 +174,7 @@ class test_truncate12(wttest.WiredTigerTestCase):
         # or running on FLCS where it isn't supported.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t' or self.trunc_with_remove:
+        if self.value_format == '8t' or self.trunc_with_remove:
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate13.py b/src/third_party/wiredtiger/test/suite/test_truncate13.py
index e61eb314d35..e17e750af20 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate13.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate13.py
@@ -223,11 +223,7 @@ class test_truncate13(wttest.WiredTigerTestCase):
         # (Or if we are running with trunc_with_remove.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t' or self.trunc_with_remove:
+        if self.value_format == '8t' or self.trunc_with_remove:
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate14.py b/src/third_party/wiredtiger/test/suite/test_truncate14.py
index 3df9a669371..5b037718ced 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate14.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate14.py
@@ -213,11 +213,7 @@ class test_truncate14(wttest.WiredTigerTestCase):
         # (Except if we're running with trunc_with_remove.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.trunc_with_remove:
+        if self.trunc_with_remove:
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate15.py b/src/third_party/wiredtiger/test/suite/test_truncate15.py
index 2b34f14f113..f9b8c5c62e2 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate15.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate15.py
@@ -163,11 +163,7 @@ class test_truncate15(wttest.WiredTigerTestCase):
         # support, so assert we didn't.
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t':
+        if self.value_format == '8t':
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate16.py b/src/third_party/wiredtiger/test/suite/test_truncate16.py
index 0a5ecdc16eb..0b192bf26cf 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate16.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate16.py
@@ -145,11 +145,7 @@ class test_truncate16(wttest.WiredTigerTestCase):
         # or running on FLCS where it isn't supported.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t' or self.trunc_with_remove:
+        if self.value_format == '8t' or self.trunc_with_remove:
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
@@ -172,11 +168,7 @@ class test_truncate16(wttest.WiredTigerTestCase):
         # (But not if we weren't fast-deleting.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         read_deleted = stat_cursor[stat.conn.cache_read_deleted][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t' or self.trunc_with_remove:
+        if self.value_format == '8t' or self.trunc_with_remove:
             self.assertEqual(read_deleted, 0)
         else:
             self.assertEqual(read_deleted, 1)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate17.py b/src/third_party/wiredtiger/test/suite/test_truncate17.py
index d47e94151dd..8e15b4c5de8 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate17.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate17.py
@@ -168,11 +168,7 @@ class test_truncate17(wttest.WiredTigerTestCase):
         # or running on FLCS where it isn't supported.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t' or self.trunc_with_remove:
+        if self.value_format == '8t' or self.trunc_with_remove:
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate18.py b/src/third_party/wiredtiger/test/suite/test_truncate18.py
index 6e60aef0e8b..cc2ff934fb2 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate18.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate18.py
@@ -169,11 +169,7 @@ class test_truncate18(wttest.WiredTigerTestCase):
         # or running on FLCS where it isn't supported.)
         stat_cursor = self.session.open_cursor('statistics:', None, None)
         fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
-        if self.runningHook('tiered'):
-            # There's no way the test can guess whether fast delete is possible when
-            # flush_tier calls are "randomly" inserted.
-            pass
-        elif self.value_format == '8t' or self.trunc_with_remove:
+        if self.value_format == '8t' or self.trunc_with_remove:
             self.assertEqual(fastdelete_pages, 0)
         else:
             self.assertGreater(fastdelete_pages, 0)
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate19.py b/src/third_party/wiredtiger/test/suite/test_truncate19.py
index b2bf1aef340..108ef7792c0 100755..100644
--- a/src/third_party/wiredtiger/test/suite/test_truncate19.py
+++ b/src/third_party/wiredtiger/test/suite/test_truncate19.py
@@ -56,7 +56,6 @@ class test_truncate19(wttest.WiredTigerTestCase):
         self.session.truncate(None, None, hicursor, None)
         self.session.commit_transaction()
 
-    @wttest.skip_for_hook("tiered", "test depends of sizes of associated file objects")
     def test_truncate19(self):
         uri = 'table:oplog'
         nrows = 1000000
diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py
index 174d10a8283..2419848e3c4 100755
--- a/src/third_party/wiredtiger/test/suite/wttest.py
+++ b/src/third_party/wiredtiger/test/suite/wttest.py
@@ -348,8 +348,6 @@ class WiredTigerTestCase(unittest.TestCase):
         if seedw != 0 and seedz != 0:
             WiredTigerTestCase._randomseed = True
             WiredTigerTestCase._seeds = [seedw, seedz]
-        # We don't have a lot of output, but we want to see it right away.
-        sys.stdout.reconfigure(line_buffering=True)
         WiredTigerTestCase._globalSetup = True
 
     @staticmethod
@@ -678,8 +676,6 @@ class WiredTigerTestCase(unittest.TestCase):
         os.chdir(self.testdir)
         with open('testname.txt', 'w+') as namefile:
             namefile.write(str(self) + '\n')
-        if WiredTigerTestCase._verbose >= 2:
-            print("[pid:{}]: {}: starting".format(os.getpid(), str(self)))
         self.fdSetUp()
         self._threadLocal.currentTestCase = self
         self.ignoreTearDownLogs = False
@@ -834,9 +830,6 @@ class WiredTigerTestCase(unittest.TestCase):
         self.assertEqual(ret, wiredtiger.WT_NOTFOUND)
         bkp_cursor.close()
 
-    def runningHook(self, name):
-        return name in WiredTigerTestCase.hook_names
-
     # Set a Python breakpoint.  When this function is called,
     # the python debugger will be called as described here:
     #   https://docs.python.org/3/library/pdb.html
@@ -988,11 +981,6 @@ class WiredTigerTestCase(unittest.TestCase):
         return '%x' % t
 
     def dropUntilSuccess(self, session, uri, config=None):
-        # Most test cases consider a drop, and especially a 'drop until success',
-        # to completely remove a file's artifacts, so that the name can be reused.
-        # For tiered storage, this means removing associated cloud objects.
-        if self.runningHook('tiered') and config == None:
-            config = 'force=true,remove_shared=true'
         while True:
             try:
                 session.drop(uri, config)