Import wiredtiger: 2ba19fce21cdcf23d4b17509baaf4ecfc55e4f96 from branch mongodb-master

ref: 507ac1f262..2ba19fce21
for: 6.2.0-rc0

WT-9741 Extend tiered hooks implementation to cover most cases (#8236)

17 files changed, 403 insertions, 73 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 9c4b194e1e7..5bcfd0824e6 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "507ac1f26247ec2b881572ef3f794bf8e842bf7d"
+    "commit": "2ba19fce21cdcf23d4b17509baaf4ecfc55e4f96"
 }
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index 4266a7dab2a..453c187615b 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -1907,6 +1907,17 @@ tasks:
       - func: "unit test"
         vars:
           unit_test_args: -v 2 -R cursor13 join02 join07 schema03 timestamp22
+
+  - name: unit-test-hook-tiered
+    tags: ["python"]
+    depends_on:
+    - name: compile
+    commands:
+      - func: "fetch artifacts"
+      - func: "unit test"
+        vars:
+          unit_test_args: --hook tiered
+
   # Break out Python unit tests into multiple buckets/tasks.  We have a fixed number of buckets,
   # and we use the -b option of the test/suite/run.py script to split up the tests.
 
@@ -3963,6 +3974,7 @@ buildvariants:
     - name: checkpoint-filetypes-test
     - name: unit-test-zstd
     - name: unit-test-random-seed
+    - name: unit-test-hook-tiered
     - name: spinlock-gcc-test
     - name: spinlock-pthread-adaptive-test
     - name: compile-wtperf
diff --git a/src/third_party/wiredtiger/test/suite/hook_demo.py b/src/third_party/wiredtiger/test/suite/hook_demo.py
index 113c427c8b7..227d8fb08c0 100755
--- a/src/third_party/wiredtiger/test/suite/hook_demo.py
+++ b/src/third_party/wiredtiger/test/suite/hook_demo.py
@@ -115,6 +115,12 @@ class DemoHookCreator(wthooks.WiredTigerHookCreator):
         print('Filtering: ' + str(tests))
         return tests
 
+    # If the hook wants to override some implementation of the test framework,
+    # it would need to subclass wthooks.WiredTigerHookPlatformAPI and return
+    # an object of that type here.
+    def get_platform_api(self):
+        return None
+
     def setup_hooks(self):
         tty('>> SETUP HOOKS RUN')
         orig_session_create = self.Session['create']     # gets original function
diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py
index dcc888cd689..429bee56529 100644..100755
--- a/src/third_party/wiredtiger/test/suite/hook_tiered.py
+++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py
@@ -64,20 +64,68 @@ from wttest import WiredTigerTestCase
 # Add the local storage extension whenever we call wiredtiger_open
 def wiredtiger_open_tiered(ignored_self, args):
     auth_token = "test_token"
+
+    # The bucket name, when it appears in configuration, is relative to the database home.
+    # Also build the path name to the bucket, including the home, so it can be created.
     bucket = "mybucket"
+    bucketpath = bucket
     extension_name = "dir_store"
     prefix = "pfx-"
+    curconfig = args[-1]
+    homedir = args[0]
+
+    testcase = WiredTigerTestCase.currentTestCase()
+
+    # If there is already tiered storage enabled, we shouldn't enable it here.
+    # We might attempt to let the wiredtiger_open complete without alteration,
+    # however, we alter several other API methods that would do weird things with
+    # a different tiered_storage configuration. So better to skip the test entirely.
+    if 'tiered_storage=' in curconfig:
+        testcase.skipTest("cannot run tiered hook on a test that already uses tiered storage")
+
+    # Similarly if this test is already set up to run tiered vs non-tiered scenario, let's
+    # not get in the way.
+    if hasattr(testcase, 'tiered_conn_config'):
+        testcase.skipTest("cannot run tiered hook on a test that already includes TieredConfigMixin")
+
+    if 'in_memory=true' in curconfig:
+        testcase.skipTest("cannot run tiered hook on a test that is in-memory")
+
+    # Mark this test as readonly, but don't disallow it.  See testcase_is_readonly().
+    if 'readonly=true' in curconfig:
+        testcase._readonlyTieredTest = True
+
+    if homedir != None:
+        bucketpath = os.path.join(homedir, bucketpath)
     extension_libs = WiredTigerTestCase.findExtension('storage_sources', extension_name)
     if len(extension_libs) == 0:
         raise Exception(extension_name + ' storage source extension not found')
 
-    if not os.path.exists(bucket):
-        os.mkdir(bucket)
-    tier_string = ',tiered_storage=(auth_token=%s,' % auth_token + \
+    if not os.path.exists(bucketpath):
+        os.mkdir(bucketpath)
+
+    tier_string = ',tiered_storage=(' + \
+      'auth_token=%s,' % auth_token + \
       'bucket=%s,' % bucket + \
       'bucket_prefix=%s,' % prefix + \
-      'name=%s),' % extension_name + \
-      'extensions=[\"%s\"],' % extension_libs[0]
+      'name=%s)' % extension_name
+
+    # Build the extension strings, we'll need to merge it with any extensions
+    # already in the configuration.
+    ext_string = 'extensions=['
+    start = curconfig.find(ext_string)
+    if start >= 0:
+        end = curconfig.find(']', start)
+        if end < 0:
+            raise Exception('hook_tiered: bad extensions in config \"%s\"' % curconfig)
+        ext_string = curconfig[start: end]
+
+    tier_string += ',' + ext_string + ',\"%s\"]' % extension_libs[0]
+
+    # The current implementation of flush_tier cannot complete until a new checkpoint has completed.
+    # Single threaded tests without a checkpoint thread would hang, so have WT do the checkpoint
+    # during the flush_tier call.
+    tier_string += ',debug_mode=(flush_checkpoint=true),'
 
     args = list(args)         # convert from a readonly tuple to a writeable list
     args[-1] += tier_string   # Modify the list
@@ -87,23 +135,33 @@ def wiredtiger_open_tiered(ignored_self, args):
 
     return args
 
+# We want readonly tests to run with tiered storage, since it is possible to do readonly
+# operations.  This function is called for two purposes:
+#  - when readonly is enabled, we don't want to do flush_tier calls.
+#  - normally the hook silently removes other (not supported) calls, like compact/rename/salvage.
+#    Except that some tests enable readonly and call these functions, expecting an exception.
+#    So for these "modifying" APIs, we want to actually do the operation (but only when readonly).
+def testcase_is_readonly():
+    testcase = WiredTigerTestCase.currentTestCase()
+    return getattr(testcase, '_readonlyTieredTest', False)
+
+def testcase_has_failed():
+    testcase = WiredTigerTestCase.currentTestCase()
+    return testcase.failed()
+
 # Called to replace Connection.close
 # Insert a call to flush_tier before closing connection.
 def connection_close_replace(orig_connection_close, connection_self, config):
-    s = connection_self.open_session(None)
-    s.flush_tier(None)
-    s.close()
-    ret = orig_connection_close(connection_self, config)
-    return ret
+    # We cannot call flush_tier on a readonly connection.
+    # Likewise we should not call flush_tier if the test case has failed,
+    # and the connection is being closed at the end of the run after the failure.
+    # Otherwise, diagnosing the original failure may be troublesome.
+    if not testcase_is_readonly() and not testcase_has_failed():
+        s = connection_self.open_session(None)
+        s.flush_tier(None)
+        s.close()
 
-# Called to replace Session.alter
-def session_alter_replace(orig_session_alter, session_self, uri, config):
-    # Alter isn't implemented for tiered tables.  Only call it if this can't be the uri
-    # of a tiered table.  Note this isn't a precise match for when we did/didn't create
-    # a tiered table, but we don't have the create config around to check.
-    ret = 0
-    if not uri.startswith("table:"):
-        ret = orig_session_alter(session_self, uri, config)
+    ret = orig_connection_close(connection_self, config)
     return ret
 
 # Called to replace Session.checkpoint.
@@ -113,17 +171,21 @@ def session_checkpoint_replace(orig_session_checkpoint, session_self, config):
     ret = orig_session_checkpoint(session_self, config)
     if ret != 0:
         return ret
-    WiredTigerTestCase.verbose(None, 3,
-        '    Calling flush_tier() after checkpoint')
-    return session_self.flush_tier(None)
+
+    # We cannot call flush_tier on a readonly connection.
+    if not testcase_is_readonly():
+        WiredTigerTestCase.verbose(None, 3,
+            '    Calling flush_tier() after checkpoint')
+        return session_self.flush_tier(None)
 
 # Called to replace Session.compact
 def session_compact_replace(orig_session_compact, session_self, uri, config):
     # Compact isn't implemented for tiered tables.  Only call it if this can't be the uri
     # of a tiered table.  Note this isn't a precise match for when we did/didn't create
     # a tiered table, but we don't have the create config around to check.
+    # We want readonly connections to do the real call, see comment in testcase_is_readonly.
     ret = 0
-    if not uri.startswith("table:"):
+    if not uri.startswith("table:") or testcase_is_readonly():
         ret = orig_session_compact(session_self, uri, config)
     return ret
 
@@ -136,7 +198,9 @@ def session_create_replace(orig_session_create, session_self, uri, config):
 
     # If the test isn't creating a table (i.e., it's a column store or lsm) create it as a
     # "local only" object.  Otherwise we get tiered storage from the connection defaults.
-    if not uri.startswith("table:") or "key_format=r" in new_config or "type=lsm" in new_config:
+    # We want readonly connections to do the real call, see comment in testcase_is_readonly.
+    # FIXME-WT-9832 Column store testing should be allowed with this hook.
+    if not uri.startswith("table:") or "key_format=r" in new_config or "type=lsm" in new_config or testcase_is_readonly():
         new_config = new_config + ',tiered_storage=(name=none)'
 
     WiredTigerTestCase.verbose(None, 3,
@@ -144,23 +208,26 @@ def session_create_replace(orig_session_create, session_self, uri, config):
     ret = orig_session_create(session_self, uri, new_config)
     return ret
 
-# Called to replace Session.drop
-def session_drop_replace(orig_session_drop, session_self, uri, config):
-    # Drop isn't implemented for tiered tables.  Only call it if this can't be the uri
-    # of a tiered table.  Note this isn't a precise match for when we did/didn't create
-    # a tiered table, but we don't have the create config around to check.
-    ret = 0
-    if not uri.startswith("table:"):
-        ret = orig_session_drop(session_self, uri, config)
-    return ret
+# FIXME-WT-9785
+# Called to replace Session.open_cursor. This is needed to skip tests that
+# do statistics on (tiered) table data sources, as that is not yet supported.
+def session_open_cursor_replace(orig_session_open_cursor, session_self, uri, dupcursor, config):
+    if uri != None and (uri.startswith("statistics:table:") or uri.startswith("statistics:file:")):
+        testcase = WiredTigerTestCase.currentTestCase()
+        testcase.skipTest("statistics on tiered tables not yet implemented")
+    if uri != None and uri.startswith("backup:"):
+        testcase = WiredTigerTestCase.currentTestCase()
+        testcase.skipTest("backup on tiered tables not yet implemented")
+    return orig_session_open_cursor(session_self, uri, dupcursor, config)
 
 # Called to replace Session.rename
 def session_rename_replace(orig_session_rename, session_self, uri, newuri, config):
     # Rename isn't implemented for tiered tables.  Only call it if this can't be the uri
     # of a tiered table.  Note this isn't a precise match for when we did/didn't create
     # a tiered table, but we don't have the create config around to check.
+    # We want readonly connections to do the real call, see comment in testcase_is_readonly.
     ret = 0
-    if not uri.startswith("table:"):
+    if not uri.startswith("table:") or testcase_is_readonly():
         ret = orig_session_rename(session_self, uri, newuri, config)
     return ret
 
@@ -169,8 +236,9 @@ def session_salvage_replace(orig_session_salvage, session_self, uri, config):
     # Salvage isn't implemented for tiered tables.  Only call it if this can't be the uri
     # of a tiered table.  Note this isn't a precise match for when we did/didn't create
     # a tiered table, but we don't have the create config around to check.
+    # We want readonly connections to do the real call, see comment in testcase_is_readonly.
     ret = 0
-    if not uri.startswith("table:"):
+    if not uri.startswith("table:") or testcase_is_readonly():
         ret = orig_session_salvage(session_self, uri, config)
     return ret
 
@@ -179,8 +247,9 @@ def session_verify_replace(orig_session_verify, session_self, uri, config):
     # Verify isn't implemented for tiered tables.  Only call it if this can't be the uri
     # of a tiered table.  Note this isn't a precise match for when we did/didn't create
     # a tiered table, but we don't have the create config around to check.
+    # We want readonly connections to do the real call, see comment in testcase_is_readonly.
     ret = 0
-    if not uri.startswith("table:"):
+    if not uri.startswith("table:") or testcase_is_readonly():
         ret = orig_session_verify(session_self, uri, config)
     return ret
 
@@ -190,21 +259,95 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator):
     def __init__(self, arg=0):
         # Caller can specify an optional command-line argument.  We're not using it
         # now, but this is where it would show up.
-        return
+
+        # Override some platform APIs
+        self.platform_api = TieredPlatformAPI()
 
     # Is this test one we should skip?
     def skip_test(self, test):
         # Skip any test that contains one of these strings as a substring
         skip = ["backup",               # Can't backup a tiered table
+                "env01",                # Using environment variable to set WT home
+                "config02",             # Using environment variable to set WT home
                 "cursor13_ckpt",        # Checkpoint tests with cached cursors
-                "cursor13_drops",       # Tests that require working drop implementation
                 "cursor13_dup",         # More cursor cache tests
                 "cursor13_reopens",     # More cursor cache tests
+                "inmem",                # In memory tests don't make sense with tiered storage
                 "lsm",                  # If the test name tells us it uses lsm ignore it
                 "modify_smoke_recover", # Copying WT dir doesn't copy the bucket directory
+                "salvage01",            # Salvage tests directly name files ending in ".wt"
                 "test_config_json",     # create replacement can't handle a json config string
                 "test_cursor_big",      # Cursor caching verified with stats
-                "tiered"]
+                "tiered",               # Tiered tests already do tiering.
+                "verify_api_75pct_null",# Test damages file, then reopens connection (flushes tier)
+                                        # so local file is undamaged
+
+                # FIXME-WT-9809 The following failures should be triaged and potentially
+                # individually reticketed.
+
+                # This first group currently cause severe errors, where Python crashes,
+                # whether from internal assertion or other causes.
+                "test_bug003.test_bug003",   # crashes in connection close after opening bulk cursor.
+                "test_bug024.test_bug024",
+                "test_durable_ts03.test_durable_ts03",
+                "test_rollback_to_stable20.test_rollback_to_stable",
+                "test_stat_log01_readonly.test_stat_log01_readonly",
+                "test_stat_log02.test_stats_log_on_json_with_tables",
+                "test_txn02.test_ops",
+                "test_upgrade.test_upgrade",
+
+                # This group fail within Python for various, sometimes unknown, reasons.
+                "test_bug018.test_bug018",
+                "test_checkpoint.test_checkpoint",
+                "test_checkpoint_target.test_checkpoint_target",
+                "test_checkpoint_snapshot02.test_checkpoint_snapshot_with_txnid_and_timestamp",
+                "test_compat05.test_compat05",
+                "test_config05.test_too_many_sessions",
+                "test_config09.test_config09",
+                "test_drop.test_drop",
+                "test_empty.test_empty",     # looks at wt file names and uses column store
+                "test_encrypt06.test_encrypt",
+                "test_encrypt07.test_salvage_api",
+                "test_encrypt07.test_salvage_api_damaged",
+                "test_encrypt07.test_salvage_process_damaged",
+                "test_export01.test_export_restart",
+                "test_hs21.test_hs",
+                "test_import04.test_table_import",
+                "test_import09.test_import_table_repair",
+                "test_import09.test_import_table_repair",
+                "test_import11.test_file_import",
+                "test_import11.test_file_import",
+                "test_join03.test_join",
+                "test_join07.test_join_string",
+                "test_jsondump02.test_json_all_bytes",
+                "test_metadata_cursor02.test_missing",
+                "test_prepare02.test_prepare_session_operations",
+                "test_prepare_hs03.test_prepare_hs",
+                "test_prepare_hs03.test_prepare_hs",
+                "test_rename.test_rename",
+                "test_rollback_to_stable09.test_rollback_to_stable",
+                "test_rollback_to_stable28.test_update_restore_evict_recovery",
+                "test_rollback_to_stable34.test_rollback_to_stable",
+                "test_rollback_to_stable35.test_rollback_to_stable",
+                "test_rollback_to_stable36.test_rollback_to_stable",
+                "test_sweep03.test_disable_idle_timeout_drop",
+                "test_sweep03.test_disable_idle_timeout_drop_force",
+                "test_truncate01.test_truncate_cursor_end",
+                "test_truncate01.test_truncate_timestamp",
+                "test_truncate01.test_truncate_uri",
+                "test_truncate10.test_truncate10",
+                "test_truncate12.test_truncate12",
+                "test_truncate13.test_truncate",
+                "test_truncate14.test_truncate",
+                "test_truncate16.test_truncate16",
+                "test_truncate18.test_truncate18",
+                "test_truncate15.test_truncate15",
+                "test_txn22.test_corrupt_meta",
+                "test_verbose01.test_verbose_single",
+                "test_verbose02.test_verbose_single",
+                "test_verify2.test_verify_ckpt",
+                ]
+
         for item in skip:
             if item in str(test):
                 return True
@@ -216,15 +359,14 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator):
         new_tests.addTests([t for t in tests if not self.skip_test(t)])
         return new_tests
 
+    def get_platform_api(self):
+        return self.platform_api
+
     def setup_hooks(self):
         orig_connection_close = self.Connection['close']
         self.Connection['close'] = (wthooks.HOOK_REPLACE, lambda s, config=None:
           connection_close_replace(orig_connection_close, s, config))
 
-        orig_session_alter = self.Session['alter']
-        self.Session['alter'] =  (wthooks.HOOK_REPLACE, lambda s, uri, config=None:
-          session_alter_replace(orig_session_alter, s, uri, config))
-
         orig_session_compact = self.Session['compact']
         self.Session['compact'] =  (wthooks.HOOK_REPLACE, lambda s, uri, config=None:
           session_compact_replace(orig_session_compact, s, uri, config))
@@ -233,9 +375,9 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator):
         self.Session['create'] =  (wthooks.HOOK_REPLACE, lambda s, uri, config=None:
           session_create_replace(orig_session_create, s, uri, config))
 
-        orig_session_drop = self.Session['drop']
-        self.Session['drop'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None:
-          session_drop_replace(orig_session_drop, s, uri, config))
+        orig_session_open_cursor = self.Session['open_cursor']
+        self.Session['open_cursor'] = (wthooks.HOOK_REPLACE, lambda s, uri, todup=None, config=None:
+          session_open_cursor_replace(orig_session_open_cursor, s, uri, todup, config))
 
         orig_session_rename = self.Session['rename']
         self.Session['rename'] = (wthooks.HOOK_REPLACE, lambda s, uri, newuri, config=None:
@@ -251,6 +393,21 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator):
 
         self.wiredtiger['wiredtiger_open'] = (wthooks.HOOK_ARGS, wiredtiger_open_tiered)
 
+# Override some platform APIs for this hook.
+class TieredPlatformAPI(wthooks.WiredTigerHookPlatformAPI):
+    def tableExists(self, name):
+        for i in range(1, 9):
+            tablename = name + "-000000000{}.wtobj".format(i)
+            if os.path.exists(tablename):
+                return True
+        return False
+
+    def initialFileName(self, uri):
+        if uri.startswith('table:'):
+            return uri[6:] + '-0000000001.wtobj'
+        else:
+            return wthooks.DefaultPlatformAPI.initialFileName(uri)
+
 # Every hook file must have a top level initialize function,
 # returning a list of WiredTigerHook objects.
 def initialize(arg):
diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py
index 64c452578af..258ecb390dc 100755
--- a/src/third_party/wiredtiger/test/suite/run.py
+++ b/src/third_party/wiredtiger/test/suite/run.py
@@ -127,8 +127,10 @@ Options:\n\
   -r N    | --random-sample N    randomly sort scenarios to be run, then\n\
                                  execute every Nth (2<=N<=1000) scenario.\n\
   -s N    | --scenario N         use scenario N (N can be symbolic, number, or\n\
-                                 list of numbers and ranges in the form 1,3-5,7)\n\
+                                 list of numbers and ranges in the form 1,3-5,7),\n\
+                                 and -1 matches tests with no scenarios.\n\
   -t      | --timestamp          name WT_TEST according to timestamp\n\
+            --timeout N          have any test that exceeds N seconds throw an error.\n\
   -v N    | --verbose N          set verboseness to N (0<=N<=3, default=1)\n\
   -i      | --ignore-stdout      dont fail on unexpected stdout or stderr\n\
   -R      | --randomseed         run with random seeds for generates random numbers\n\
@@ -188,7 +190,13 @@ def parse_int_list(str):
             scenario = int(bounds[0])
             ret[scenario] = True
             continue
-        if len(bounds) == 2 and bounds[0].isdigit() and bounds[1].isdigit():
+        elif len(bounds) == 2 and len(bounds[0]) == 0 and bounds[1].isdigit():
+            # It's a negative number.  We indicate "has no scenarios" by -1, anything else is not allowed.
+            if r == '-1':
+                scenario = -1
+                ret[scenario] = True
+                continue
+        elif len(bounds) == 2 and bounds[0].isdigit() and bounds[1].isdigit():
             # It's two numbers separated by a dash.
             for scenario in range(int(bounds[0]), int(bounds[1]) + 1):
                 ret[scenario] = True
@@ -198,13 +206,19 @@ def parse_int_list(str):
     return ret
 
 def restrictScenario(testcases, restrict):
+    # Inner function to see if test case matches a scenario list
+    def scenarioMatch(testcase, scenario_list):
+        matchint = -1
+        if hasattr(testcase, 'scenario_number'):
+            matchint = int(testcase.scenario_number)
+        return matchint in scenario_list
+
     if restrict == '':
         return testcases
     else:
         scenarios = parse_int_list(restrict)
         if scenarios is not None:
-            return [t for t in testcases
-                if hasattr(t, 'scenario_number') and t.scenario_number in scenarios]
+            return [t for t in testcases if scenarioMatch(t, scenarios)]
         else:
             return [t for t in testcases
                 if hasattr(t, 'scenario_name') and t.scenario_name == restrict]
@@ -348,6 +362,7 @@ if __name__ == '__main__':
     args = sys.argv[1:]
     testargs = []
     hook_names = []
+    timeout = 0
     # Generate a random string to use as a prefix for the tiered test objects to group them under
     # the same test run.
     ss_random_prefix = str(random.randrange(1, 2147483646))
@@ -443,6 +458,12 @@ if __name__ == '__main__':
             if option == '-timestamp' or option == 't':
                 timestamp = True
                 continue
+            if option == '-timeout':
+                if timeout != 0 or len(args) == 0:
+                    usage()
+                    sys.exit(2)
+                timeout = int(args.pop(0))
+                continue
             if option == '-verbose' or option == 'v':
                 if len(args) == 0:
                     usage()
@@ -571,15 +592,11 @@ if __name__ == '__main__':
     # That way, verbose printing can be done at the class definition level.
     wttest.WiredTigerTestCase.globalSetup(preserve, removeAtStart, timestamp, gdbSub, lldbSub,
                                           verbose, wt_builddir, dirarg, longtest, zstdtest,
-                                          ignoreStdout, seedw, seedz, hookmgr, ss_random_prefix)
+                                          ignoreStdout, seedw, seedz, hookmgr, ss_random_prefix,
+                                          timeout)
 
     # Without any tests listed as arguments, do discovery
     if len(testargs) == 0:
-        if scenario != '':
-            sys.stderr.write(
-                'run.py: specifying a scenario requires a test name\n')
-            usage()
-            sys.exit(2)
         from discover import defaultTestLoader as loader
         suites = loader.discover(suitedir)
 
@@ -599,12 +616,13 @@ if __name__ == '__main__':
         suites = sorted(suites, key=lambda c: str(list(c)[0]))
         if configfile != None:
             suites = configApply(suites, configfile, configwrite)
-        tests.addTests(restrictScenario(generate_scenarios(suites), ''))
+        tests.addTests(restrictScenario(generate_scenarios(suites), scenario))
     else:
         for arg in testargs:
             testsFromArg(tests, loader, arg, scenario)
 
     tests = hookmgr.filter_tests(tests)
+
     # Shuffle the tests and create a new suite containing every Nth test from
     # the original suite
     if random_sample > 0:
diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py
index 6cd61ce1e9a..2aab7e4f186 100755
--- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py
+++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py
@@ -228,6 +228,12 @@ class suite_subprocess:
         outfilename=None, errfilename=None, closeconn=True,
         reopensession=True, failure=False):
 
+        # FIXME-WT-9808:
+        # The tiered hook silently interjects tiered configuration and extensions,
+        # these are not yet dealt with when running the external 'wt' process.
+        if 'tiered' in self.hook_names:
+            self.skipTest("runWt is not yet supported with tiering")
+
         # Close the connection to guarantee everything is flushed, and that
         # we can open it from another process.
         if closeconn:
diff --git a/src/third_party/wiredtiger/test/suite/test_bug006.py b/src/third_party/wiredtiger/test/suite/test_bug006.py
index 835a3caa506..b978c62c487 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_bug006.py
+++ b/src/third_party/wiredtiger/test/suite/test_bug006.py
@@ -43,6 +43,8 @@ class test_bug006(wttest.WiredTigerTestCase):
     ])
 
     def test_bug006(self):
+        if 'tiered' in self.hook_names:
+            self.skipTest("negative tests for session APIs like drop do not work in tiered storage")
         uri = self.uri + self.name
         self.session.create(uri, 'value_format=S,key_format=S')
         cursor = self.session.open_cursor(uri, None)
diff --git a/src/third_party/wiredtiger/test/suite/test_drop.py b/src/third_party/wiredtiger/test/suite/test_drop.py
index e9f0aaea4ea..2edc48122f7 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_drop.py
+++ b/src/third_party/wiredtiger/test/suite/test_drop.py
@@ -86,6 +86,8 @@ class test_drop(wttest.WiredTigerTestCase):
 
     # Test drop of a non-existent object: force succeeds, without force fails.
     def test_drop_dne(self):
+        if 'tiered' in self.hook_names:
+            self.skipTest("negative tests for drop do not work in tiered storage")
         uri = self.uri + self.name
         cguri = 'colgroup:' + self.name
         idxuri = 'index:' + self.name + ':indexname'
diff --git a/src/third_party/wiredtiger/test/suite/test_dupc.py b/src/third_party/wiredtiger/test/suite/test_dupc.py
index 890af88c63c..0c17d20aaba 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_dupc.py
+++ b/src/third_party/wiredtiger/test/suite/test_dupc.py
@@ -69,6 +69,11 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase):
         cursor.close()
 
     def test_duplicate_cursor(self):
+        # FIXME-WT-9815:
+        # Using column store ComplexDataStore after SimpleDataStore doesn't work, see ticket.
+        if 'tiered' in self.hook_names:
+            self.skipTest("this test does not yet work with tiered storage")
+
         uri = self.uri + self.name
 
         # A simple, one-file file or table object.
diff --git a/src/third_party/wiredtiger/test/suite/test_util04.py b/src/third_party/wiredtiger/test/suite/test_util04.py
index 66ef04c1b9c..752f25ce43f 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_util04.py
+++ b/src/third_party/wiredtiger/test/suite/test_util04.py
@@ -43,10 +43,10 @@ class test_util04(wttest.WiredTigerTestCase, suite_subprocess):
         params = 'key_format=S,value_format=S'
         self.session.create('table:' + self.tablename, params)
 
-        self.assertTrue(os.path.exists(self.tablename + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename))
         self.runWt(["drop", "table:" + self.tablename])
 
-        self.assertFalse(os.path.exists(self.tablename + ".wt"))
+        self.assertFalse(self.tableExists(self.tablename))
         self.assertRaises(wiredtiger.WiredTigerError, lambda:
             self.session.open_cursor('table:' + self.tablename, None, None))
 
diff --git a/src/third_party/wiredtiger/test/suite/test_util14.py b/src/third_party/wiredtiger/test/suite/test_util14.py
index d46b7ca468f..d9a4b8368cd 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_util14.py
+++ b/src/third_party/wiredtiger/test/suite/test_util14.py
@@ -42,7 +42,7 @@ class test_util14(wttest.WiredTigerTestCase, suite_subprocess):
         """
         params = 'key_format=S,value_format=S'
         self.session.create('table:' + self.tablename, params)
-        self.assertTrue(os.path.exists(self.tablename + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename))
         cursor = self.session.open_cursor('table:' + self.tablename, None, None)
         for i in range(0, self.nentries):
             cursor[str(i)] = str(i)
@@ -55,7 +55,7 @@ class test_util14(wttest.WiredTigerTestCase, suite_subprocess):
         """
         outfile="outfile.txt"
         errfile="errfile.txt"
-        self.assertTrue(os.path.exists(self.tablename + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename))
         self.runWt(["read", 'table:' + self.tablename, 'NoMatch'],
             outfilename=outfile, errfilename=errfile, failure=True)
         self.check_empty_file(outfile)
diff --git a/src/third_party/wiredtiger/test/suite/test_util15.py b/src/third_party/wiredtiger/test/suite/test_util15.py
index c7a28488937..6830834697a 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_util15.py
+++ b/src/third_party/wiredtiger/test/suite/test_util15.py
@@ -41,7 +41,7 @@ class test_util15(wttest.WiredTigerTestCase, suite_subprocess):
         """
         params = 'key_format=S,value_format=S'
         self.session.create('table:' + self.tablename, params)
-        self.assertTrue(os.path.exists(self.tablename + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename))
 
         """
         Alter access pattern and confirm
diff --git a/src/third_party/wiredtiger/test/suite/test_util16.py b/src/third_party/wiredtiger/test/suite/test_util16.py
index 5960e120e9e..648777f5242 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_util16.py
+++ b/src/third_party/wiredtiger/test/suite/test_util16.py
@@ -43,14 +43,14 @@ class test_util16(wttest.WiredTigerTestCase, suite_subprocess):
         """
         params = 'key_format=S,value_format=S'
         self.session.create('table:' + self.tablename, params)
-        self.assertTrue(os.path.exists(self.tablename + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename))
         cursor = self.session.open_cursor('table:' + self.tablename, None, None)
         for i in range(0, self.nentries):
             cursor[str(i)] = str(i)
         cursor.close()
 
         self.runWt(["rename", "table:" + self.tablename, "table:" + self.tablename2])
-        self.assertTrue(os.path.exists(self.tablename2 + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename2))
         cursor = self.session.open_cursor('table:' + self.tablename2, None, None)
         count = 0
         while cursor.next() == 0:
@@ -59,7 +59,7 @@ class test_util16(wttest.WiredTigerTestCase, suite_subprocess):
         self.assertEquals(self.nentries, count)
 
         self.runWt(["rename", "table:" + self.tablename2, "table:" + self.tablename])
-        self.assertTrue(os.path.exists(self.tablename + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename))
         cursor = self.session.open_cursor('table:' + self.tablename, None, None)
         count = 0
         while cursor.next() == 0:
diff --git a/src/third_party/wiredtiger/test/suite/test_util17.py b/src/third_party/wiredtiger/test/suite/test_util17.py
index c7edebce82c..a4065d3c8a4 100644..100755
--- a/src/third_party/wiredtiger/test/suite/test_util17.py
+++ b/src/third_party/wiredtiger/test/suite/test_util17.py
@@ -45,7 +45,7 @@ class test_util17(wttest.WiredTigerTestCase, suite_subprocess):
         outfile = "wt-stat.out"
         expected_string = "cursor: cursor create calls="
         self.session.create('table:' + self.tablename, params)
-        self.assertTrue(os.path.exists(self.tablename + ".wt"))
+        self.assertTrue(self.tableExists(self.tablename))
         self.runWt(["stat"], outfilename=outfile)
         self.check_file_contains(outfile, expected_string)
 
diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py
index af0ae6e5f11..f48afab1854 100755
--- a/src/third_party/wiredtiger/test/suite/test_verify.py
+++ b/src/third_party/wiredtiger/test/suite/test_verify.py
@@ -36,6 +36,13 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess):
     tablename = 'test_verify.a'
     nentries = 1000
 
+    # Returns the .wt file extension, or in the case
+    # of tiered storage, builds the .wtobj object name.
+    # Assumes that no checkpoints are done, so we
+    # are on the first object.
+    def file_name(self, name):
+        return self.initialFileName('table:' + name)
+
     def populate(self, tablename):
         """
         Insert some simple entries into the table
@@ -74,7 +81,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess):
         if self.conn != None:
             self.conn.close()
             self.conn = None
-        filename = tablename + ".wt"
+        filename = self.file_name(tablename)
 
         filesize = os.path.getsize(filename)
         position = (filesize * pct) // 100
diff --git a/src/third_party/wiredtiger/test/suite/wthooks.py b/src/third_party/wiredtiger/test/suite/wthooks.py
index 80d375f32f9..3cdd2e32475 100755
--- a/src/third_party/wiredtiger/test/suite/wthooks.py
+++ b/src/third_party/wiredtiger/test/suite/wthooks.py
@@ -37,7 +37,7 @@ from __future__ import print_function
 
 from importlib import import_module
 from abc import ABC, abstractmethod
-import wiredtiger
+import wiredtiger, os
 
 # Three kinds of hooks available:
 HOOK_REPLACE = 1     # replace the call with the hook function
@@ -82,6 +82,14 @@ def tty(message):
 #      A hook function that replaces an API function will have the same args as the function
 #      it replaces (but there is a trick to give it additional context if needed -
 #      see session_create_replace in hook_demo.py).
+#
+#  Hook Platform API:
+#      A set of utility functions used by WiredTigerTestCase or other parts of the test framework
+#      that may differ according to platform.  Rather than have hook specific implementations in the
+#      test framework, the "platform API" is implemented by any hook that wants to override it.
+#      Currently the hook specific implementation is all or nothing, in the future we may allow
+#      subsets of the hook platform API to be implemented.
+
 
 # For every API function altered, there is one of these objects
 # stashed in the <class>._<api_name>_hooks attribute.
@@ -135,6 +143,7 @@ def hooked_function(self, orig_func, hook_info_name, *args):
 class WiredTigerHookManager(object):
     def __init__(self, hooknames = []):
         self.hooks = []
+        self.platform_api = None
         names_seen = []
         for name in hooknames:
             # The hooks are indicated as "somename=arg" or simply "somename".
@@ -159,8 +168,20 @@ class WiredTigerHookManager(object):
             except:
                 print('Cannot import hook: ' + name + ', check file ' + modname + '.py')
                 raise
+        self.hook_names = tuple(names_seen)
         for hook in self.hooks:
             hook.setup_hooks()
+            api = hook.get_platform_api()   # can return None
+            if api:
+                # We currently don't allow multiple platforms to create their own API,
+                # but this could be relaxed. Imagine that hooks implement subsets of the
+                # API. We could create an ordered list, and try each platform_api in turn.
+                if self.platform_api:
+                    raise Exception('Running multiple hooks, each with their own platform API, ' +
+                                    'is not implemented')
+                self.platform_api = api
+        if self.platform_api == None:
+            self.platform_api = DefaultPlatformAPI()
 
     def add_hook(self, clazz, method_name, hook_type, hook_func):
         if not hasattr(clazz, method_name):
@@ -219,6 +240,12 @@ class WiredTigerHookManager(object):
             tests = hook.filter_tests(tests)
         return tests
 
+    def get_hook_names(self):
+        return self.hook_names
+
+    def get_platform_api(self):
+        return self.platform_api
+
 class HookCreatorProxy(object):
     def __init__(self, hookmgr, clazz):
         self.hookmgr = hookmgr
@@ -256,3 +283,27 @@ class WiredTigerHookCreator(ABC):
     def setup_hooks(self):
         """Set up all hooks using add_*_hook methods."""
         return
+
+class WiredTigerHookPlatformAPI(ABC):
+    @abstractmethod
+    def tableExists(self, name):
+        """Return boolean if local files exist for the table with the given base name"""
+        pass
+
+    @abstractmethod
+    def initialFileName(self, uri):
+        """The first local backing file name created for this URI."""
+        pass
+
+class DefaultPlatformAPI(WiredTigerHookPlatformAPI):
+    def tableExists(self, name):
+        tablename = name + ".wt"
+        return os.path.exists(tablename)
+
+    def initialFileName(self, uri):
+        if uri.startswith('table:'):
+            return uri[6:] + '.wt'
+        elif uri.startswith('file:'):
+            return uri[5:]
+        else:
+            raise Exception('bad uri')
diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py
index cd528412132..dbe152e65df 100755
--- a/src/third_party/wiredtiger/test/suite/wttest.py
+++ b/src/third_party/wiredtiger/test/suite/wttest.py
@@ -42,9 +42,37 @@ except ImportError:
     import unittest
 
 from contextlib import contextmanager
-import errno, glob, os, re, shutil, sys, time, traceback
+import errno, glob, os, re, shutil, sys, threading, time, traceback
 import wiredtiger, wtscenario, wthooks
 
+# Use as "with timeout(seconds): ....". Argument of 0 means no timeout,
+# and only available (with non-zero argument) on Unix systems.
+class timeout(object):
+    def __init__(self, seconds=0):
+        self.seconds = seconds
+        self.prev_handler = None
+
+    def signal_handler(self, signum, frame):
+        raise(TimeoutError('time for test exceeded {} seconds'.format(self.seconds)))
+
+    def __enter__(self):
+        if self.seconds != 0:
+            try:
+                import signal     # This will fail on non-Unix systems.
+                self.prev_handler = signal.signal(signal.SIGALRM, self.signal_handler)
+                signal.alarm(self.seconds)
+            except Exception as e:
+                raise Exception('The --timeout option is not available on this system: ' + str(e))
+
+    def __exit__(self, typ, value, traceback):
+        if self.seconds != 0:
+            try:
+                import signal     # This will fail on non-Unix systems.
+                signal.alarm(0)
+                signal.signal(signal.SIGALRM, self.prev_handler)
+            except Exception as e:
+                raise Exception('The --timeout option is not available on this system: ' + str(e))
+
 def shortenWithEllipsis(s, maxlen):
     if len(s) > maxlen:
         s = s[0:maxlen-3] + '...'
@@ -185,6 +213,11 @@ class WiredTigerTestCase(unittest.TestCase):
     _printOnceSeen = {}
     _ttyDescriptor = None   # set this early, to allow tty() to be called any time.
 
+    # We store the current test case in thread local storage.  There are
+    # certain odd cases where this is useful, like hooks, where we don't
+    # have any notion of the current test case.
+    _threadLocal = threading.local()
+
     # rollbacks_allowed can be overridden to permit more or fewer retries on rollback errors.
     # We retry tests that get rollback errors in a way that is mostly invisible.
     # There is a visible difference in that the rollback error's stack trace is recorded
@@ -211,7 +244,7 @@ class WiredTigerTestCase(unittest.TestCase):
     def globalSetup(preserveFiles = False, removeAtStart = True, useTimestamp = False,
                     gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None,
                     longtest = False, zstdtest = False, ignoreStdout = False, seedw = 0, seedz = 0, 
-                    hookmgr = None, ss_random_prefix = 0):
+                    hookmgr = None, ss_random_prefix = 0, timeout = 0):
         WiredTigerTestCase._preserveFiles = preserveFiles
         d = 'WT_TEST' if dirarg == None else dirarg
         if useTimestamp:
@@ -241,9 +274,11 @@ class WiredTigerTestCase(unittest.TestCase):
         WiredTigerTestCase._ss_random_prefix = ss_random_prefix
         WiredTigerTestCase._retriesAfterRollback = 0
         WiredTigerTestCase._testsRun = 0
+        WiredTigerTestCase._timeout = timeout
         if hookmgr == None:
             hookmgr = wthooks.WiredTigerHookManager()
         WiredTigerTestCase._hookmgr = hookmgr
+        WiredTigerTestCase.hook_names = hookmgr.get_hook_names()
         if seedw != 0 and seedz != 0:
             WiredTigerTestCase._randomseed = True
             WiredTigerTestCase._seeds = [seedw, seedz]
@@ -263,6 +298,10 @@ class WiredTigerTestCase(unittest.TestCase):
             raise Exception('Retries from WT_ROLLBACK in test suite: {}/{}, see {} for stack traces'.format(
                 totalRetries, totalTestsRun, WiredTigerTestCase._resultFileName))
 
+    @staticmethod
+    def currentTestCase():
+        return getattr(WiredTigerTestCase._threadLocal, 'currentTestCase', None)
+
     def fdSetUp(self):
         self.captureout = CapturedFd('stdout.txt', 'standard output')
         self.captureerr = CapturedFd('stderr.txt', 'error output')
@@ -283,6 +322,20 @@ class WiredTigerTestCase(unittest.TestCase):
         self.skipped = False
         if not self._globalSetup:
             WiredTigerTestCase.globalSetup()
+        self.platform_api = WiredTigerTestCase._hookmgr.get_platform_api()
+
+    # Platform specific functions (may be overridden by hooks):
+
+    # Return true if file(s) for the table with the given base name exist in the file system.
+    # This may have a different implementation when running under certain hooks.
+    def tableExists(self, name):
+        return self.platform_api.tableExists(name)
+
+    # The first filename for this URI.  In the tiered storage
+    # world, this makes a difference, every flush tier creates a
+    # This may have a different implementation when running under certain hooks.
+    def initialFileName(self, name):
+        return self.platform_api.initialFileName(name)
 
     def __str__(self):
         # when running with scenarios, if the number_scenarios() method
@@ -325,8 +378,9 @@ class WiredTigerTestCase(unittest.TestCase):
         WiredTigerTestCase._testsRun += 1
         while not finished and rollbacksAllowed >= 0:
             try:
-                method()
-                finished = True
+                with timeout(WiredTigerTestCase._timeout):
+                    method()
+                    finished = True
             except wiredtiger.WiredTigerRollbackError:
                 WiredTigerTestCase._retriesAfterRollback += 1
                 self.prexception(sys.exc_info())
@@ -515,6 +569,7 @@ class WiredTigerTestCase(unittest.TestCase):
             self.prhead('started in ' + self.testdir, True)
         # tearDown needs connections list, set it here in case the open fails.
         self._connections = []
+        self._failed = None   # set to True/False during teardown.
         self.origcwd = os.getcwd()
         shutil.rmtree(self.testdir, ignore_errors=True)
         if os.path.exists(self.testdir):
@@ -524,6 +579,7 @@ class WiredTigerTestCase(unittest.TestCase):
         with open('testname.txt', 'w+') as namefile:
             namefile.write(str(self) + '\n')
         self.fdSetUp()
+        self._threadLocal.currentTestCase = self
         # tearDown needs a conn field, set it here in case the open fails.
         self.conn = None
         try:
@@ -576,7 +632,8 @@ class WiredTigerTestCase(unittest.TestCase):
         failure = self.list2reason(result, 'failures')
         exc_failure = (sys.exc_info() != (None, None, None))
 
-        passed = not error and not failure and not exc_failure
+        self._failed = error or failure or exc_failure
+        passed = not self._failed
 
         # Download the files from the S3 bucket for tiered tests if the test fails or preserve is
         # turned on.
@@ -618,6 +675,8 @@ class WiredTigerTestCase(unittest.TestCase):
         else:
             self.pr('preserving directory ' + self.testdir)
 
+        self._threadLocal.currentTestCase = None
+
         elapsed = time.time() - self.starttime
         if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2:
             print("%s: %.2f seconds" % (str(self), elapsed))
@@ -628,6 +687,11 @@ class WiredTigerTestCase(unittest.TestCase):
         if WiredTigerTestCase._verbose > 2:
             self.prhead('TEST COMPLETED')
 
+    # Returns None if testcase is running.  If during (or after) tearDown,
+    # will return True or False depending if the test case failed.
+    def failed(self):
+        return self._failed
+
     def backup(self, backup_dir, session=None):
         if session is None:
             session = self.session