diff options
17 files changed, 403 insertions, 73 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 9c4b194e1e7..5bcfd0824e6 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "507ac1f26247ec2b881572ef3f794bf8e842bf7d" + "commit": "2ba19fce21cdcf23d4b17509baaf4ecfc55e4f96" } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 4266a7dab2a..453c187615b 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1907,6 +1907,17 @@ tasks: - func: "unit test" vars: unit_test_args: -v 2 -R cursor13 join02 join07 schema03 timestamp22 + + - name: unit-test-hook-tiered + tags: ["python"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "unit test" + vars: + unit_test_args: --hook tiered + # Break out Python unit tests into multiple buckets/tasks. We have a fixed number of buckets, # and we use the -b option of the test/suite/run.py script to split up the tests. @@ -3963,6 +3974,7 @@ buildvariants: - name: checkpoint-filetypes-test - name: unit-test-zstd - name: unit-test-random-seed + - name: unit-test-hook-tiered - name: spinlock-gcc-test - name: spinlock-pthread-adaptive-test - name: compile-wtperf diff --git a/src/third_party/wiredtiger/test/suite/hook_demo.py b/src/third_party/wiredtiger/test/suite/hook_demo.py index 113c427c8b7..227d8fb08c0 100755 --- a/src/third_party/wiredtiger/test/suite/hook_demo.py +++ b/src/third_party/wiredtiger/test/suite/hook_demo.py @@ -115,6 +115,12 @@ class DemoHookCreator(wthooks.WiredTigerHookCreator): print('Filtering: ' + str(tests)) return tests + # If the hook wants to override some implementation of the test framework, + # it would need to subclass wthooks.WiredTigerHookPlatformAPI and return + # an object of that type here. + def get_platform_api(self): + return None + def setup_hooks(self): tty('>> SETUP HOOKS RUN') orig_session_create = self.Session['create'] # gets original function diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py index dcc888cd689..429bee56529 100644..100755 --- a/src/third_party/wiredtiger/test/suite/hook_tiered.py +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -64,20 +64,68 @@ from wttest import WiredTigerTestCase # Add the local storage extension whenever we call wiredtiger_open def wiredtiger_open_tiered(ignored_self, args): auth_token = "test_token" + + # The bucket name, when it appears in configuration, is relative to the database home. + # Also build the path name to the bucket, including the home, so it can be created. bucket = "mybucket" + bucketpath = bucket extension_name = "dir_store" prefix = "pfx-" + curconfig = args[-1] + homedir = args[0] + + testcase = WiredTigerTestCase.currentTestCase() + + # If there is already tiered storage enabled, we shouldn't enable it here. + # We might attempt to let the wiredtiger_open complete without alteration, + # however, we alter several other API methods that would do weird things with + # a different tiered_storage configuration. So better to skip the test entirely. + if 'tiered_storage=' in curconfig: + testcase.skipTest("cannot run tiered hook on a test that already uses tiered storage") + + # Similarly if this test is already set up to run tiered vs non-tiered scenario, let's + # not get in the way. + if hasattr(testcase, 'tiered_conn_config'): + testcase.skipTest("cannot run tiered hook on a test that already includes TieredConfigMixin") + + if 'in_memory=true' in curconfig: + testcase.skipTest("cannot run tiered hook on a test that is in-memory") + + # Mark this test as readonly, but don't disallow it. See testcase_is_readonly(). + if 'readonly=true' in curconfig: + testcase._readonlyTieredTest = True + + if homedir != None: + bucketpath = os.path.join(homedir, bucketpath) extension_libs = WiredTigerTestCase.findExtension('storage_sources', extension_name) if len(extension_libs) == 0: raise Exception(extension_name + ' storage source extension not found') - if not os.path.exists(bucket): - os.mkdir(bucket) - tier_string = ',tiered_storage=(auth_token=%s,' % auth_token + \ + if not os.path.exists(bucketpath): + os.mkdir(bucketpath) + + tier_string = ',tiered_storage=(' + \ + 'auth_token=%s,' % auth_token + \ 'bucket=%s,' % bucket + \ 'bucket_prefix=%s,' % prefix + \ - 'name=%s),' % extension_name + \ - 'extensions=[\"%s\"],' % extension_libs[0] + 'name=%s)' % extension_name + + # Build the extension strings, we'll need to merge it with any extensions + # already in the configuration. + ext_string = 'extensions=[' + start = curconfig.find(ext_string) + if start >= 0: + end = curconfig.find(']', start) + if end < 0: + raise Exception('hook_tiered: bad extensions in config \"%s\"' % curconfig) + ext_string = curconfig[start: end] + + tier_string += ',' + ext_string + ',\"%s\"]' % extension_libs[0] + + # The current implementation of flush_tier cannot complete until a new checkpoint has completed. + # Single threaded tests without a checkpoint thread would hang, so have WT do the checkpoint + # during the flush_tier call. + tier_string += ',debug_mode=(flush_checkpoint=true),' args = list(args) # convert from a readonly tuple to a writeable list args[-1] += tier_string # Modify the list @@ -87,23 +135,33 @@ def wiredtiger_open_tiered(ignored_self, args): return args +# We want readonly tests to run with tiered storage, since it is possible to do readonly +# operations. This function is called for two purposes: +# - when readonly is enabled, we don't want to do flush_tier calls. +# - normally the hook silently removes other (not supported) calls, like compact/rename/salvage. +# Except that some tests enable readonly and call these functions, expecting an exception. +# So for these "modifying" APIs, we want to actually do the operation (but only when readonly). +def testcase_is_readonly(): + testcase = WiredTigerTestCase.currentTestCase() + return getattr(testcase, '_readonlyTieredTest', False) + +def testcase_has_failed(): + testcase = WiredTigerTestCase.currentTestCase() + return testcase.failed() + # Called to replace Connection.close # Insert a call to flush_tier before closing connection. def connection_close_replace(orig_connection_close, connection_self, config): - s = connection_self.open_session(None) - s.flush_tier(None) - s.close() - ret = orig_connection_close(connection_self, config) - return ret + # We cannot call flush_tier on a readonly connection. + # Likewise we should not call flush_tier if the test case has failed, + # and the connection is being closed at the end of the run after the failure. + # Otherwise, diagnosing the original failure may be troublesome. + if not testcase_is_readonly() and not testcase_has_failed(): + s = connection_self.open_session(None) + s.flush_tier(None) + s.close() -# Called to replace Session.alter -def session_alter_replace(orig_session_alter, session_self, uri, config): - # Alter isn't implemented for tiered tables. Only call it if this can't be the uri - # of a tiered table. Note this isn't a precise match for when we did/didn't create - # a tiered table, but we don't have the create config around to check. - ret = 0 - if not uri.startswith("table:"): - ret = orig_session_alter(session_self, uri, config) + ret = orig_connection_close(connection_self, config) return ret # Called to replace Session.checkpoint. @@ -113,17 +171,21 @@ def session_checkpoint_replace(orig_session_checkpoint, session_self, config): ret = orig_session_checkpoint(session_self, config) if ret != 0: return ret - WiredTigerTestCase.verbose(None, 3, - ' Calling flush_tier() after checkpoint') - return session_self.flush_tier(None) + + # We cannot call flush_tier on a readonly connection. + if not testcase_is_readonly(): + WiredTigerTestCase.verbose(None, 3, + ' Calling flush_tier() after checkpoint') + return session_self.flush_tier(None) # Called to replace Session.compact def session_compact_replace(orig_session_compact, session_self, uri, config): # Compact isn't implemented for tiered tables. Only call it if this can't be the uri # of a tiered table. Note this isn't a precise match for when we did/didn't create # a tiered table, but we don't have the create config around to check. + # We want readonly connections to do the real call, see comment in testcase_is_readonly. ret = 0 - if not uri.startswith("table:"): + if not uri.startswith("table:") or testcase_is_readonly(): ret = orig_session_compact(session_self, uri, config) return ret @@ -136,7 +198,9 @@ def session_create_replace(orig_session_create, session_self, uri, config): # If the test isn't creating a table (i.e., it's a column store or lsm) create it as a # "local only" object. Otherwise we get tiered storage from the connection defaults. - if not uri.startswith("table:") or "key_format=r" in new_config or "type=lsm" in new_config: + # We want readonly connections to do the real call, see comment in testcase_is_readonly. + # FIXME-WT-9832 Column store testing should be allowed with this hook. + if not uri.startswith("table:") or "key_format=r" in new_config or "type=lsm" in new_config or testcase_is_readonly(): new_config = new_config + ',tiered_storage=(name=none)' WiredTigerTestCase.verbose(None, 3, @@ -144,23 +208,26 @@ def session_create_replace(orig_session_create, session_self, uri, config): ret = orig_session_create(session_self, uri, new_config) return ret -# Called to replace Session.drop -def session_drop_replace(orig_session_drop, session_self, uri, config): - # Drop isn't implemented for tiered tables. Only call it if this can't be the uri - # of a tiered table. Note this isn't a precise match for when we did/didn't create - # a tiered table, but we don't have the create config around to check. - ret = 0 - if not uri.startswith("table:"): - ret = orig_session_drop(session_self, uri, config) - return ret +# FIXME-WT-9785 +# Called to replace Session.open_cursor. This is needed to skip tests that +# do statistics on (tiered) table data sources, as that is not yet supported. +def session_open_cursor_replace(orig_session_open_cursor, session_self, uri, dupcursor, config): + if uri != None and (uri.startswith("statistics:table:") or uri.startswith("statistics:file:")): + testcase = WiredTigerTestCase.currentTestCase() + testcase.skipTest("statistics on tiered tables not yet implemented") + if uri != None and uri.startswith("backup:"): + testcase = WiredTigerTestCase.currentTestCase() + testcase.skipTest("backup on tiered tables not yet implemented") + return orig_session_open_cursor(session_self, uri, dupcursor, config) # Called to replace Session.rename def session_rename_replace(orig_session_rename, session_self, uri, newuri, config): # Rename isn't implemented for tiered tables. Only call it if this can't be the uri # of a tiered table. Note this isn't a precise match for when we did/didn't create # a tiered table, but we don't have the create config around to check. + # We want readonly connections to do the real call, see comment in testcase_is_readonly. ret = 0 - if not uri.startswith("table:"): + if not uri.startswith("table:") or testcase_is_readonly(): ret = orig_session_rename(session_self, uri, newuri, config) return ret @@ -169,8 +236,9 @@ def session_salvage_replace(orig_session_salvage, session_self, uri, config): # Salvage isn't implemented for tiered tables. Only call it if this can't be the uri # of a tiered table. Note this isn't a precise match for when we did/didn't create # a tiered table, but we don't have the create config around to check. + # We want readonly connections to do the real call, see comment in testcase_is_readonly. ret = 0 - if not uri.startswith("table:"): + if not uri.startswith("table:") or testcase_is_readonly(): ret = orig_session_salvage(session_self, uri, config) return ret @@ -179,8 +247,9 @@ def session_verify_replace(orig_session_verify, session_self, uri, config): # Verify isn't implemented for tiered tables. Only call it if this can't be the uri # of a tiered table. Note this isn't a precise match for when we did/didn't create # a tiered table, but we don't have the create config around to check. + # We want readonly connections to do the real call, see comment in testcase_is_readonly. ret = 0 - if not uri.startswith("table:"): + if not uri.startswith("table:") or testcase_is_readonly(): ret = orig_session_verify(session_self, uri, config) return ret @@ -190,21 +259,95 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): def __init__(self, arg=0): # Caller can specify an optional command-line argument. We're not using it # now, but this is where it would show up. - return + + # Override some platform APIs + self.platform_api = TieredPlatformAPI() # Is this test one we should skip? def skip_test(self, test): # Skip any test that contains one of these strings as a substring skip = ["backup", # Can't backup a tiered table + "env01", # Using environment variable to set WT home + "config02", # Using environment variable to set WT home "cursor13_ckpt", # Checkpoint tests with cached cursors - "cursor13_drops", # Tests that require working drop implementation "cursor13_dup", # More cursor cache tests "cursor13_reopens", # More cursor cache tests + "inmem", # In memory tests don't make sense with tiered storage "lsm", # If the test name tells us it uses lsm ignore it "modify_smoke_recover", # Copying WT dir doesn't copy the bucket directory + "salvage01", # Salvage tests directly name files ending in ".wt" "test_config_json", # create replacement can't handle a json config string "test_cursor_big", # Cursor caching verified with stats - "tiered"] + "tiered", # Tiered tests already do tiering. + "verify_api_75pct_null",# Test damages file, then reopens connection (flushes tier) + # so local file is undamaged + + # FIXME-WT-9809 The following failures should be triaged and potentially + # individually reticketed. + + # This first group currently cause severe errors, where Python crashes, + # whether from internal assertion or other causes. + "test_bug003.test_bug003", # crashes in connection close after opening bulk cursor. + "test_bug024.test_bug024", + "test_durable_ts03.test_durable_ts03", + "test_rollback_to_stable20.test_rollback_to_stable", + "test_stat_log01_readonly.test_stat_log01_readonly", + "test_stat_log02.test_stats_log_on_json_with_tables", + "test_txn02.test_ops", + "test_upgrade.test_upgrade", + + # This group fail within Python for various, sometimes unknown, reasons. + "test_bug018.test_bug018", + "test_checkpoint.test_checkpoint", + "test_checkpoint_target.test_checkpoint_target", + "test_checkpoint_snapshot02.test_checkpoint_snapshot_with_txnid_and_timestamp", + "test_compat05.test_compat05", + "test_config05.test_too_many_sessions", + "test_config09.test_config09", + "test_drop.test_drop", + "test_empty.test_empty", # looks at wt file names and uses column store + "test_encrypt06.test_encrypt", + "test_encrypt07.test_salvage_api", + "test_encrypt07.test_salvage_api_damaged", + "test_encrypt07.test_salvage_process_damaged", + "test_export01.test_export_restart", + "test_hs21.test_hs", + "test_import04.test_table_import", + "test_import09.test_import_table_repair", + "test_import09.test_import_table_repair", + "test_import11.test_file_import", + "test_import11.test_file_import", + "test_join03.test_join", + "test_join07.test_join_string", + "test_jsondump02.test_json_all_bytes", + "test_metadata_cursor02.test_missing", + "test_prepare02.test_prepare_session_operations", + "test_prepare_hs03.test_prepare_hs", + "test_prepare_hs03.test_prepare_hs", + "test_rename.test_rename", + "test_rollback_to_stable09.test_rollback_to_stable", + "test_rollback_to_stable28.test_update_restore_evict_recovery", + "test_rollback_to_stable34.test_rollback_to_stable", + "test_rollback_to_stable35.test_rollback_to_stable", + "test_rollback_to_stable36.test_rollback_to_stable", + "test_sweep03.test_disable_idle_timeout_drop", + "test_sweep03.test_disable_idle_timeout_drop_force", + "test_truncate01.test_truncate_cursor_end", + "test_truncate01.test_truncate_timestamp", + "test_truncate01.test_truncate_uri", + "test_truncate10.test_truncate10", + "test_truncate12.test_truncate12", + "test_truncate13.test_truncate", + "test_truncate14.test_truncate", + "test_truncate16.test_truncate16", + "test_truncate18.test_truncate18", + "test_truncate15.test_truncate15", + "test_txn22.test_corrupt_meta", + "test_verbose01.test_verbose_single", + "test_verbose02.test_verbose_single", + "test_verify2.test_verify_ckpt", + ] + for item in skip: if item in str(test): return True @@ -216,15 +359,14 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): new_tests.addTests([t for t in tests if not self.skip_test(t)]) return new_tests + def get_platform_api(self): + return self.platform_api + def setup_hooks(self): orig_connection_close = self.Connection['close'] self.Connection['close'] = (wthooks.HOOK_REPLACE, lambda s, config=None: connection_close_replace(orig_connection_close, s, config)) - orig_session_alter = self.Session['alter'] - self.Session['alter'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: - session_alter_replace(orig_session_alter, s, uri, config)) - orig_session_compact = self.Session['compact'] self.Session['compact'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_compact_replace(orig_session_compact, s, uri, config)) @@ -233,9 +375,9 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): self.Session['create'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_create_replace(orig_session_create, s, uri, config)) - orig_session_drop = self.Session['drop'] - self.Session['drop'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: - session_drop_replace(orig_session_drop, s, uri, config)) + orig_session_open_cursor = self.Session['open_cursor'] + self.Session['open_cursor'] = (wthooks.HOOK_REPLACE, lambda s, uri, todup=None, config=None: + session_open_cursor_replace(orig_session_open_cursor, s, uri, todup, config)) orig_session_rename = self.Session['rename'] self.Session['rename'] = (wthooks.HOOK_REPLACE, lambda s, uri, newuri, config=None: @@ -251,6 +393,21 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): self.wiredtiger['wiredtiger_open'] = (wthooks.HOOK_ARGS, wiredtiger_open_tiered) +# Override some platform APIs for this hook. +class TieredPlatformAPI(wthooks.WiredTigerHookPlatformAPI): + def tableExists(self, name): + for i in range(1, 9): + tablename = name + "-000000000{}.wtobj".format(i) + if os.path.exists(tablename): + return True + return False + + def initialFileName(self, uri): + if uri.startswith('table:'): + return uri[6:] + '-0000000001.wtobj' + else: + return wthooks.DefaultPlatformAPI.initialFileName(uri) + # Every hook file must have a top level initialize function, # returning a list of WiredTigerHook objects. def initialize(arg): diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 64c452578af..258ecb390dc 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -127,8 +127,10 @@ Options:\n\ -r N | --random-sample N randomly sort scenarios to be run, then\n\ execute every Nth (2<=N<=1000) scenario.\n\ -s N | --scenario N use scenario N (N can be symbolic, number, or\n\ - list of numbers and ranges in the form 1,3-5,7)\n\ + list of numbers and ranges in the form 1,3-5,7),\n\ + and -1 matches tests with no scenarios.\n\ -t | --timestamp name WT_TEST according to timestamp\n\ + --timeout N have any test that exceeds N seconds throw an error.\n\ -v N | --verbose N set verboseness to N (0<=N<=3, default=1)\n\ -i | --ignore-stdout dont fail on unexpected stdout or stderr\n\ -R | --randomseed run with random seeds for generates random numbers\n\ @@ -188,7 +190,13 @@ def parse_int_list(str): scenario = int(bounds[0]) ret[scenario] = True continue - if len(bounds) == 2 and bounds[0].isdigit() and bounds[1].isdigit(): + elif len(bounds) == 2 and len(bounds[0]) == 0 and bounds[1].isdigit(): + # It's a negative number. We indicate "has no scenarios" by -1, anything else is not allowed. + if r == '-1': + scenario = -1 + ret[scenario] = True + continue + elif len(bounds) == 2 and bounds[0].isdigit() and bounds[1].isdigit(): # It's two numbers separated by a dash. for scenario in range(int(bounds[0]), int(bounds[1]) + 1): ret[scenario] = True @@ -198,13 +206,19 @@ def parse_int_list(str): return ret def restrictScenario(testcases, restrict): + # Inner function to see if test case matches a scenario list + def scenarioMatch(testcase, scenario_list): + matchint = -1 + if hasattr(testcase, 'scenario_number'): + matchint = int(testcase.scenario_number) + return matchint in scenario_list + if restrict == '': return testcases else: scenarios = parse_int_list(restrict) if scenarios is not None: - return [t for t in testcases - if hasattr(t, 'scenario_number') and t.scenario_number in scenarios] + return [t for t in testcases if scenarioMatch(t, scenarios)] else: return [t for t in testcases if hasattr(t, 'scenario_name') and t.scenario_name == restrict] @@ -348,6 +362,7 @@ if __name__ == '__main__': args = sys.argv[1:] testargs = [] hook_names = [] + timeout = 0 # Generate a random string to use as a prefix for the tiered test objects to group them under # the same test run. ss_random_prefix = str(random.randrange(1, 2147483646)) @@ -443,6 +458,12 @@ if __name__ == '__main__': if option == '-timestamp' or option == 't': timestamp = True continue + if option == '-timeout': + if timeout != 0 or len(args) == 0: + usage() + sys.exit(2) + timeout = int(args.pop(0)) + continue if option == '-verbose' or option == 'v': if len(args) == 0: usage() @@ -571,15 +592,11 @@ if __name__ == '__main__': # That way, verbose printing can be done at the class definition level. wttest.WiredTigerTestCase.globalSetup(preserve, removeAtStart, timestamp, gdbSub, lldbSub, verbose, wt_builddir, dirarg, longtest, zstdtest, - ignoreStdout, seedw, seedz, hookmgr, ss_random_prefix) + ignoreStdout, seedw, seedz, hookmgr, ss_random_prefix, + timeout) # Without any tests listed as arguments, do discovery if len(testargs) == 0: - if scenario != '': - sys.stderr.write( - 'run.py: specifying a scenario requires a test name\n') - usage() - sys.exit(2) from discover import defaultTestLoader as loader suites = loader.discover(suitedir) @@ -599,12 +616,13 @@ if __name__ == '__main__': suites = sorted(suites, key=lambda c: str(list(c)[0])) if configfile != None: suites = configApply(suites, configfile, configwrite) - tests.addTests(restrictScenario(generate_scenarios(suites), '')) + tests.addTests(restrictScenario(generate_scenarios(suites), scenario)) else: for arg in testargs: testsFromArg(tests, loader, arg, scenario) tests = hookmgr.filter_tests(tests) + # Shuffle the tests and create a new suite containing every Nth test from # the original suite if random_sample > 0: diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index 6cd61ce1e9a..2aab7e4f186 100755 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -228,6 +228,12 @@ class suite_subprocess: outfilename=None, errfilename=None, closeconn=True, reopensession=True, failure=False): + # FIXME-WT-9808: + # The tiered hook silently interjects tiered configuration and extensions, + # these are not yet dealt with when running the external 'wt' process. + if 'tiered' in self.hook_names: + self.skipTest("runWt is not yet supported with tiering") + # Close the connection to guarantee everything is flushed, and that # we can open it from another process. if closeconn: diff --git a/src/third_party/wiredtiger/test/suite/test_bug006.py b/src/third_party/wiredtiger/test/suite/test_bug006.py index 835a3caa506..b978c62c487 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_bug006.py +++ b/src/third_party/wiredtiger/test/suite/test_bug006.py @@ -43,6 +43,8 @@ class test_bug006(wttest.WiredTigerTestCase): ]) def test_bug006(self): + if 'tiered' in self.hook_names: + self.skipTest("negative tests for session APIs like drop do not work in tiered storage") uri = self.uri + self.name self.session.create(uri, 'value_format=S,key_format=S') cursor = self.session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_drop.py b/src/third_party/wiredtiger/test/suite/test_drop.py index e9f0aaea4ea..2edc48122f7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_drop.py +++ b/src/third_party/wiredtiger/test/suite/test_drop.py @@ -86,6 +86,8 @@ class test_drop(wttest.WiredTigerTestCase): # Test drop of a non-existent object: force succeeds, without force fails. def test_drop_dne(self): + if 'tiered' in self.hook_names: + self.skipTest("negative tests for drop do not work in tiered storage") uri = self.uri + self.name cguri = 'colgroup:' + self.name idxuri = 'index:' + self.name + ':indexname' diff --git a/src/third_party/wiredtiger/test/suite/test_dupc.py b/src/third_party/wiredtiger/test/suite/test_dupc.py index 890af88c63c..0c17d20aaba 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_dupc.py +++ b/src/third_party/wiredtiger/test/suite/test_dupc.py @@ -69,6 +69,11 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): cursor.close() def test_duplicate_cursor(self): + # FIXME-WT-9815: + # Using column store ComplexDataStore after SimpleDataStore doesn't work, see ticket. + if 'tiered' in self.hook_names: + self.skipTest("this test does not yet work with tiered storage") + uri = self.uri + self.name # A simple, one-file file or table object. diff --git a/src/third_party/wiredtiger/test/suite/test_util04.py b/src/third_party/wiredtiger/test/suite/test_util04.py index 66ef04c1b9c..752f25ce43f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util04.py +++ b/src/third_party/wiredtiger/test/suite/test_util04.py @@ -43,10 +43,10 @@ class test_util04(wttest.WiredTigerTestCase, suite_subprocess): params = 'key_format=S,value_format=S' self.session.create('table:' + self.tablename, params) - self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.assertTrue(self.tableExists(self.tablename)) self.runWt(["drop", "table:" + self.tablename]) - self.assertFalse(os.path.exists(self.tablename + ".wt")) + self.assertFalse(self.tableExists(self.tablename)) self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor('table:' + self.tablename, None, None)) diff --git a/src/third_party/wiredtiger/test/suite/test_util14.py b/src/third_party/wiredtiger/test/suite/test_util14.py index d46b7ca468f..d9a4b8368cd 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util14.py +++ b/src/third_party/wiredtiger/test/suite/test_util14.py @@ -42,7 +42,7 @@ class test_util14(wttest.WiredTigerTestCase, suite_subprocess): """ params = 'key_format=S,value_format=S' self.session.create('table:' + self.tablename, params) - self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.assertTrue(self.tableExists(self.tablename)) cursor = self.session.open_cursor('table:' + self.tablename, None, None) for i in range(0, self.nentries): cursor[str(i)] = str(i) @@ -55,7 +55,7 @@ class test_util14(wttest.WiredTigerTestCase, suite_subprocess): """ outfile="outfile.txt" errfile="errfile.txt" - self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.assertTrue(self.tableExists(self.tablename)) self.runWt(["read", 'table:' + self.tablename, 'NoMatch'], outfilename=outfile, errfilename=errfile, failure=True) self.check_empty_file(outfile) diff --git a/src/third_party/wiredtiger/test/suite/test_util15.py b/src/third_party/wiredtiger/test/suite/test_util15.py index c7a28488937..6830834697a 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util15.py +++ b/src/third_party/wiredtiger/test/suite/test_util15.py @@ -41,7 +41,7 @@ class test_util15(wttest.WiredTigerTestCase, suite_subprocess): """ params = 'key_format=S,value_format=S' self.session.create('table:' + self.tablename, params) - self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.assertTrue(self.tableExists(self.tablename)) """ Alter access pattern and confirm diff --git a/src/third_party/wiredtiger/test/suite/test_util16.py b/src/third_party/wiredtiger/test/suite/test_util16.py index 5960e120e9e..648777f5242 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util16.py +++ b/src/third_party/wiredtiger/test/suite/test_util16.py @@ -43,14 +43,14 @@ class test_util16(wttest.WiredTigerTestCase, suite_subprocess): """ params = 'key_format=S,value_format=S' self.session.create('table:' + self.tablename, params) - self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.assertTrue(self.tableExists(self.tablename)) cursor = self.session.open_cursor('table:' + self.tablename, None, None) for i in range(0, self.nentries): cursor[str(i)] = str(i) cursor.close() self.runWt(["rename", "table:" + self.tablename, "table:" + self.tablename2]) - self.assertTrue(os.path.exists(self.tablename2 + ".wt")) + self.assertTrue(self.tableExists(self.tablename2)) cursor = self.session.open_cursor('table:' + self.tablename2, None, None) count = 0 while cursor.next() == 0: @@ -59,7 +59,7 @@ class test_util16(wttest.WiredTigerTestCase, suite_subprocess): self.assertEquals(self.nentries, count) self.runWt(["rename", "table:" + self.tablename2, "table:" + self.tablename]) - self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.assertTrue(self.tableExists(self.tablename)) cursor = self.session.open_cursor('table:' + self.tablename, None, None) count = 0 while cursor.next() == 0: diff --git a/src/third_party/wiredtiger/test/suite/test_util17.py b/src/third_party/wiredtiger/test/suite/test_util17.py index c7edebce82c..a4065d3c8a4 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util17.py +++ b/src/third_party/wiredtiger/test/suite/test_util17.py @@ -45,7 +45,7 @@ class test_util17(wttest.WiredTigerTestCase, suite_subprocess): outfile = "wt-stat.out" expected_string = "cursor: cursor create calls=" self.session.create('table:' + self.tablename, params) - self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.assertTrue(self.tableExists(self.tablename)) self.runWt(["stat"], outfilename=outfile) self.check_file_contains(outfile, expected_string) diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py index af0ae6e5f11..f48afab1854 100755 --- a/src/third_party/wiredtiger/test/suite/test_verify.py +++ b/src/third_party/wiredtiger/test/suite/test_verify.py @@ -36,6 +36,13 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_verify.a' nentries = 1000 + # Returns the .wt file extension, or in the case + # of tiered storage, builds the .wtobj object name. + # Assumes that no checkpoints are done, so we + # are on the first object. + def file_name(self, name): + return self.initialFileName('table:' + name) + def populate(self, tablename): """ Insert some simple entries into the table @@ -74,7 +81,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): if self.conn != None: self.conn.close() self.conn = None - filename = tablename + ".wt" + filename = self.file_name(tablename) filesize = os.path.getsize(filename) position = (filesize * pct) // 100 diff --git a/src/third_party/wiredtiger/test/suite/wthooks.py b/src/third_party/wiredtiger/test/suite/wthooks.py index 80d375f32f9..3cdd2e32475 100755 --- a/src/third_party/wiredtiger/test/suite/wthooks.py +++ b/src/third_party/wiredtiger/test/suite/wthooks.py @@ -37,7 +37,7 @@ from __future__ import print_function from importlib import import_module from abc import ABC, abstractmethod -import wiredtiger +import wiredtiger, os # Three kinds of hooks available: HOOK_REPLACE = 1 # replace the call with the hook function @@ -82,6 +82,14 @@ def tty(message): # A hook function that replaces an API function will have the same args as the function # it replaces (but there is a trick to give it additional context if needed - # see session_create_replace in hook_demo.py). +# +# Hook Platform API: +# A set of utility functions used by WiredTigerTestCase or other parts of the test framework +# that may differ according to platform. Rather than have hook specific implementations in the +# test framework, the "platform API" is implemented by any hook that wants to override it. +# Currently the hook specific implementation is all or nothing, in the future we may allow +# subsets of the hook platform API to be implemented. + # For every API function altered, there is one of these objects # stashed in the <class>._<api_name>_hooks attribute. @@ -135,6 +143,7 @@ def hooked_function(self, orig_func, hook_info_name, *args): class WiredTigerHookManager(object): def __init__(self, hooknames = []): self.hooks = [] + self.platform_api = None names_seen = [] for name in hooknames: # The hooks are indicated as "somename=arg" or simply "somename". @@ -159,8 +168,20 @@ class WiredTigerHookManager(object): except: print('Cannot import hook: ' + name + ', check file ' + modname + '.py') raise + self.hook_names = tuple(names_seen) for hook in self.hooks: hook.setup_hooks() + api = hook.get_platform_api() # can return None + if api: + # We currently don't allow multiple platforms to create their own API, + # but this could be relaxed. Imagine that hooks implement subsets of the + # API. We could create an ordered list, and try each platform_api in turn. + if self.platform_api: + raise Exception('Running multiple hooks, each with their own platform API, ' + + 'is not implemented') + self.platform_api = api + if self.platform_api == None: + self.platform_api = DefaultPlatformAPI() def add_hook(self, clazz, method_name, hook_type, hook_func): if not hasattr(clazz, method_name): @@ -219,6 +240,12 @@ class WiredTigerHookManager(object): tests = hook.filter_tests(tests) return tests + def get_hook_names(self): + return self.hook_names + + def get_platform_api(self): + return self.platform_api + class HookCreatorProxy(object): def __init__(self, hookmgr, clazz): self.hookmgr = hookmgr @@ -256,3 +283,27 @@ class WiredTigerHookCreator(ABC): def setup_hooks(self): """Set up all hooks using add_*_hook methods.""" return + +class WiredTigerHookPlatformAPI(ABC): + @abstractmethod + def tableExists(self, name): + """Return boolean if local files exist for the table with the given base name""" + pass + + @abstractmethod + def initialFileName(self, uri): + """The first local backing file name created for this URI.""" + pass + +class DefaultPlatformAPI(WiredTigerHookPlatformAPI): + def tableExists(self, name): + tablename = name + ".wt" + return os.path.exists(tablename) + + def initialFileName(self, uri): + if uri.startswith('table:'): + return uri[6:] + '.wt' + elif uri.startswith('file:'): + return uri[5:] + else: + raise Exception('bad uri') diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index cd528412132..dbe152e65df 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -42,9 +42,37 @@ except ImportError: import unittest from contextlib import contextmanager -import errno, glob, os, re, shutil, sys, time, traceback +import errno, glob, os, re, shutil, sys, threading, time, traceback import wiredtiger, wtscenario, wthooks +# Use as "with timeout(seconds): ....". Argument of 0 means no timeout, +# and only available (with non-zero argument) on Unix systems. +class timeout(object): + def __init__(self, seconds=0): + self.seconds = seconds + self.prev_handler = None + + def signal_handler(self, signum, frame): + raise(TimeoutError('time for test exceeded {} seconds'.format(self.seconds))) + + def __enter__(self): + if self.seconds != 0: + try: + import signal # This will fail on non-Unix systems. + self.prev_handler = signal.signal(signal.SIGALRM, self.signal_handler) + signal.alarm(self.seconds) + except Exception as e: + raise Exception('The --timeout option is not available on this system: ' + str(e)) + + def __exit__(self, typ, value, traceback): + if self.seconds != 0: + try: + import signal # This will fail on non-Unix systems. + signal.alarm(0) + signal.signal(signal.SIGALRM, self.prev_handler) + except Exception as e: + raise Exception('The --timeout option is not available on this system: ' + str(e)) + def shortenWithEllipsis(s, maxlen): if len(s) > maxlen: s = s[0:maxlen-3] + '...' @@ -185,6 +213,11 @@ class WiredTigerTestCase(unittest.TestCase): _printOnceSeen = {} _ttyDescriptor = None # set this early, to allow tty() to be called any time. + # We store the current test case in thread local storage. There are + # certain odd cases where this is useful, like hooks, where we don't + # have any notion of the current test case. + _threadLocal = threading.local() + # rollbacks_allowed can be overridden to permit more or fewer retries on rollback errors. # We retry tests that get rollback errors in a way that is mostly invisible. # There is a visible difference in that the rollback error's stack trace is recorded @@ -211,7 +244,7 @@ class WiredTigerTestCase(unittest.TestCase): def globalSetup(preserveFiles = False, removeAtStart = True, useTimestamp = False, gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None, longtest = False, zstdtest = False, ignoreStdout = False, seedw = 0, seedz = 0, - hookmgr = None, ss_random_prefix = 0): + hookmgr = None, ss_random_prefix = 0, timeout = 0): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg if useTimestamp: @@ -241,9 +274,11 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._ss_random_prefix = ss_random_prefix WiredTigerTestCase._retriesAfterRollback = 0 WiredTigerTestCase._testsRun = 0 + WiredTigerTestCase._timeout = timeout if hookmgr == None: hookmgr = wthooks.WiredTigerHookManager() WiredTigerTestCase._hookmgr = hookmgr + WiredTigerTestCase.hook_names = hookmgr.get_hook_names() if seedw != 0 and seedz != 0: WiredTigerTestCase._randomseed = True WiredTigerTestCase._seeds = [seedw, seedz] @@ -263,6 +298,10 @@ class WiredTigerTestCase(unittest.TestCase): raise Exception('Retries from WT_ROLLBACK in test suite: {}/{}, see {} for stack traces'.format( totalRetries, totalTestsRun, WiredTigerTestCase._resultFileName)) + @staticmethod + def currentTestCase(): + return getattr(WiredTigerTestCase._threadLocal, 'currentTestCase', None) + def fdSetUp(self): self.captureout = CapturedFd('stdout.txt', 'standard output') self.captureerr = CapturedFd('stderr.txt', 'error output') @@ -283,6 +322,20 @@ class WiredTigerTestCase(unittest.TestCase): self.skipped = False if not self._globalSetup: WiredTigerTestCase.globalSetup() + self.platform_api = WiredTigerTestCase._hookmgr.get_platform_api() + + # Platform specific functions (may be overridden by hooks): + + # Return true if file(s) for the table with the given base name exist in the file system. + # This may have a different implementation when running under certain hooks. + def tableExists(self, name): + return self.platform_api.tableExists(name) + + # The first filename for this URI. In the tiered storage + # world, this makes a difference, every flush tier creates a + # This may have a different implementation when running under certain hooks. + def initialFileName(self, name): + return self.platform_api.initialFileName(name) def __str__(self): # when running with scenarios, if the number_scenarios() method @@ -325,8 +378,9 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._testsRun += 1 while not finished and rollbacksAllowed >= 0: try: - method() - finished = True + with timeout(WiredTigerTestCase._timeout): + method() + finished = True except wiredtiger.WiredTigerRollbackError: WiredTigerTestCase._retriesAfterRollback += 1 self.prexception(sys.exc_info()) @@ -515,6 +569,7 @@ class WiredTigerTestCase(unittest.TestCase): self.prhead('started in ' + self.testdir, True) # tearDown needs connections list, set it here in case the open fails. self._connections = [] + self._failed = None # set to True/False during teardown. self.origcwd = os.getcwd() shutil.rmtree(self.testdir, ignore_errors=True) if os.path.exists(self.testdir): @@ -524,6 +579,7 @@ class WiredTigerTestCase(unittest.TestCase): with open('testname.txt', 'w+') as namefile: namefile.write(str(self) + '\n') self.fdSetUp() + self._threadLocal.currentTestCase = self # tearDown needs a conn field, set it here in case the open fails. self.conn = None try: @@ -576,7 +632,8 @@ class WiredTigerTestCase(unittest.TestCase): failure = self.list2reason(result, 'failures') exc_failure = (sys.exc_info() != (None, None, None)) - passed = not error and not failure and not exc_failure + self._failed = error or failure or exc_failure + passed = not self._failed # Download the files from the S3 bucket for tiered tests if the test fails or preserve is # turned on. @@ -618,6 +675,8 @@ class WiredTigerTestCase(unittest.TestCase): else: self.pr('preserving directory ' + self.testdir) + self._threadLocal.currentTestCase = None + elapsed = time.time() - self.starttime if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: print("%s: %.2f seconds" % (str(self), elapsed)) @@ -628,6 +687,11 @@ class WiredTigerTestCase(unittest.TestCase): if WiredTigerTestCase._verbose > 2: self.prhead('TEST COMPLETED') + # Returns None if testcase is running. If during (or after) tearDown, + # will return True or False depending if the test case failed. + def failed(self): + return self._failed + def backup(self, backup_dir, session=None): if session is None: session = self.session |