diff options
Diffstat (limited to 'buildscripts')
12 files changed, 210 insertions, 104 deletions
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_replication.yml index a2270cc4c32..d1500a90d79 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication.yml @@ -19,6 +19,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - ValidateCollections tests: true @@ -32,6 +33,7 @@ executor: # # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplDBHash - class: ValidateCollections - class: CleanupConcurrencyWorkloads diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml index db949dd4e58..4906056a9b7 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml @@ -46,6 +46,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - ValidateCollections tests: true @@ -64,6 +65,7 @@ executor: # # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplDBHash - class: ValidateCollections - class: CleanupConcurrencyWorkloads diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml index 0ce9d4506fb..dc847ce498d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml @@ -46,6 +46,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - ValidateCollections tests: true @@ -64,6 +65,7 @@ executor: # # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml index d535361c29f..c0d263d26be 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml @@ -18,9 +18,10 @@ selector: executor: archive: hooks: + - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - ValidateCollections - - CheckReplDBHashInBackground tests: true config: shell_options: @@ -32,6 +33,7 @@ executor: # # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml index 5ee8dbf0bc9..77d98918c18 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml @@ -19,6 +19,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - ValidateCollections tests: true @@ -30,6 +31,7 @@ executor: # on the secondaries, so we run the ValidateCollections hook after it to ensure we're # validating the entire contents of the collection. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplDBHash - class: ValidateCollections - class: CleanupConcurrencyWorkloads diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml index 191f31de952..89a12841254 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml @@ -19,6 +19,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - ValidateCollections tests: true @@ -30,6 +31,7 @@ executor: # on the secondaries, so we run the ValidateCollections hook after it to ensure we're # validating the entire contents of the collection. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplDBHash - class: ValidateCollections - class: CleanupConcurrencyWorkloads diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml index fabca31780d..af137e2c3ca 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml @@ -29,6 +29,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - CheckReplOplogs - ValidateCollections @@ -41,6 +42,7 @@ executor: # on the secondaries, so we run the ValidateCollections hook after it to ensure we're # validating the entire contents of the collection. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplOplogs - class: CheckReplDBHash - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml index e1ed3491d77..dca64af5445 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml @@ -23,6 +23,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - CheckReplOplogs - ValidateCollections @@ -35,6 +36,7 @@ executor: # on the secondaries, so we run the ValidateCollections hook after it to ensure we're # validating the entire contents of the collection. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplOplogs - class: CheckReplDBHash - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml index 38b0db0d9a4..00a177cc35e 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml @@ -17,6 +17,7 @@ executor: archive: hooks: - CheckReplDBHashInBackground + - ValidateCollectionsInBackground - CheckReplDBHash - CheckReplOplogs - ValidateCollections @@ -29,6 +30,7 @@ executor: # on the secondaries, so we run the ValidateCollections hook after it to ensure we're # validating the entire contents of the collection. - class: CheckReplDBHashInBackground + - class: ValidateCollectionsInBackground - class: CheckReplOplogs - class: CheckReplDBHash - class: ValidateCollections diff --git a/buildscripts/resmokelib/testing/hooks/background_job.py b/buildscripts/resmokelib/testing/hooks/background_job.py new file mode 100644 index 00000000000..39daeffb100 --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/background_job.py @@ -0,0 +1,98 @@ +"""Background job utils. Helps run a JS file repeatedly while a test suite's tests run.""" + +import sys +import threading + +from buildscripts.resmokelib.testing.hooks import jsfile + + +class _BackgroundJob(threading.Thread): # pylint: disable=too-many-instance-attributes + """A thread for running a JS file while a test is running.""" + + def __init__(self, thread_name): + """Initialize _BackgroundJob.""" + threading.Thread.__init__(self, name=thread_name) + self.daemon = True + + self._lock = threading.Lock() + self._cond = threading.Condition(self._lock) + + self._should_stop = False + self._should_resume = False + self._is_idle = True + + self._hook_test_case = None + self._test_report = None + self.exc_info = None + + def run(self): + while True: + with self._lock: + while not self._should_resume: + self._cond.wait() + + # We have the background thread set 'self._should_resume' back to false to ensure + # that 'self._hook_test_case.run_dynamic_test()' is only called once before the + # resume() method is called again. + self._should_resume = False + + if self._should_stop: + break + + # We are about to execute 'self._hook_test_case' so we mark the background thread as + # no longer being idle. + self._is_idle = False + + try: + self._hook_test_case.run_dynamic_test(self._test_report) + except: # pylint: disable=bare-except + self.exc_info = sys.exc_info() + finally: + with self._lock: + self._is_idle = True + self._cond.notify_all() + + def stop(self): + """Signal the background thread to exit, and wait until it does.""" + with self._lock: + self._should_stop = True + self.resume(hook_test_case=None, test_report=None) + self.join() + + def pause(self): # noqa: D205,D400 + """Signal the background thread that it should stop executing 'self._hook_test_case', and + wait until the current execution has finished. + """ + self._hook_test_case.signal_stop_test() + with self._lock: + while not self._is_idle: + self._cond.wait() + + def resume(self, hook_test_case, test_report): + """Instruct the background thread to start executing 'hook_test_case'.""" + self._hook_test_case = hook_test_case + self._test_report = test_report + self.exc_info = None + with self._lock: + self._should_resume = True + self._cond.notify_all() + + +class _ContinuousDynamicJSTestCase(jsfile.DynamicJSTestCase): + """A dynamic TestCase that runs a JavaScript file repeatedly.""" + + def __init__(self, *args, **kwargs): + """Initialize _ContinuousDynamicJSTestCase.""" + jsfile.DynamicJSTestCase.__init__(self, *args, **kwargs) + self._should_stop = threading.Event() + + def run_test(self): + """Execute the test repeatedly.""" + while not self._should_stop.is_set(): + jsfile.DynamicJSTestCase.run_test(self) + + def signal_stop_test(self): # noqa: D205,D400 + """Indicate to the thread executing the run_test() method that the current execution is the + last one. This method returns without waiting for the current execution to finish. + """ + self._should_stop.set() diff --git a/buildscripts/resmokelib/testing/hooks/dbhash_background.py b/buildscripts/resmokelib/testing/hooks/dbhash_background.py index 877f8c5ef1b..1faf084c58c 100644 --- a/buildscripts/resmokelib/testing/hooks/dbhash_background.py +++ b/buildscripts/resmokelib/testing/hooks/dbhash_background.py @@ -5,12 +5,11 @@ running. """ import os.path -import sys -import threading from buildscripts.resmokelib import errors from buildscripts.resmokelib.testing.hooks import jsfile from buildscripts.resmokelib.testing.testcases import interface as testcase +from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase class CheckReplDBHashInBackground(jsfile.JSHook): @@ -35,12 +34,13 @@ class CheckReplDBHashInBackground(jsfile.JSHook): server_status = client.admin.command("serverStatus") if not server_status["storageEngine"].get("supportsSnapshotReadConcern", False): self.logger.info( - "Not enabling the background thread because '%s' storage engine" - " doesn't support snapshot reads.", server_status["storageEngine"]["name"]) + "Not enabling the background check repl dbhash thread because '%s' storage" + " engine doesn't support snapshot reads.", + server_status["storageEngine"]["name"]) return - self._background_job = _BackgroundJob() - self.logger.info("Starting the background thread.") + self._background_job = _BackgroundJob("CheckReplDBHashInBackground") + self.logger.info("Starting the background check repl dbhash thread.") self._background_job.start() def after_suite(self, test_report): @@ -48,7 +48,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook): if self._background_job is None: return - self.logger.info("Stopping the background thread.") + self.logger.info("Stopping the background check repl dbhash thread.") self._background_job.stop() def before_test(self, test, test_report): @@ -60,7 +60,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook): self.logger.test_case_logger, test, self, self._js_filename, self._shell_options) hook_test_case.configure(self.fixture) - self.logger.info("Resuming the background thread.") + self.logger.info("Resuming the background check repl dbhash thread.") self._background_job.resume(hook_test_case, test_report) def after_test(self, test, test_report): # noqa: D205,D400 @@ -70,7 +70,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook): if self._background_job is None: return - self.logger.info("Pausing the background thread.") + self.logger.info("Pausing the background check repl dbhash thread.") self._background_job.pause() if self._background_job.exc_info is not None: @@ -80,98 +80,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook): # test execution to stop. raise errors.ServerFailure(self._background_job.exc_info[1].args[0]) else: - self.logger.error("Encountered an error inside the background thread.", - exc_info=self._background_job.exc_info) + self.logger.error( + "Encountered an error inside the background check repl dbhash thread.", + exc_info=self._background_job.exc_info) raise self._background_job.exc_info[1] - - -class _BackgroundJob(threading.Thread): # pylint: disable=too-many-instance-attributes - """A thread for running the dbhash check while a test is running.""" - - def __init__(self): - """Initialize _BackgroundJob.""" - threading.Thread.__init__(self, name="CheckReplDBHashInBackground") - self.daemon = True - - self._lock = threading.Lock() - self._cond = threading.Condition(self._lock) - - self._should_stop = False - self._should_resume = False - self._is_idle = True - - self._hook_test_case = None - self._test_report = None - self.exc_info = None - - def run(self): - while True: - with self._lock: - while not self._should_resume: - self._cond.wait() - - # We have the background thread set 'self._should_resume' back to false to ensure - # that 'self._hook_test_case.run_dynamic_test()' is only called once before the - # resume() method is called again. - self._should_resume = False - - if self._should_stop: - break - - # We are about to execute 'self._hook_test_case' so we mark the background thread as - # no longer being idle. - self._is_idle = False - - try: - self._hook_test_case.run_dynamic_test(self._test_report) - except: # pylint: disable=bare-except - self.exc_info = sys.exc_info() - finally: - with self._lock: - self._is_idle = True - self._cond.notify_all() - - def stop(self): - """Signal the background thread to exit, and wait until it does.""" - with self._lock: - self._should_stop = True - self.resume(hook_test_case=None, test_report=None) - self.join() - - def pause(self): # noqa: D205,D400 - """Signal the background thread that it should stop executing 'self._hook_test_case', and - wait until the current execution has finished. - """ - self._hook_test_case.signal_stop_test() - with self._lock: - while not self._is_idle: - self._cond.wait() - - def resume(self, hook_test_case, test_report): - """Instruct the background thread to start executing 'hook_test_case'.""" - self._hook_test_case = hook_test_case - self._test_report = test_report - self.exc_info = None - with self._lock: - self._should_resume = True - self._cond.notify_all() - - -class _ContinuousDynamicJSTestCase(jsfile.DynamicJSTestCase): - """A dynamic TestCase that runs a JavaScript file repeatedly.""" - - def __init__(self, *args, **kwargs): - """Initialize _ContinuousDynamicJSTestCase.""" - jsfile.DynamicJSTestCase.__init__(self, *args, **kwargs) - self._should_stop = threading.Event() - - def run_test(self): - """Execute the test repeatedly.""" - while not self._should_stop.is_set(): - jsfile.DynamicJSTestCase.run_test(self) - - def signal_stop_test(self): # noqa: D205,D400 - """Indicate to the thread executing the run_test() method that the current execution is the - last one. This method returns without waiting for the current execution to finish. - """ - self._should_stop.set() diff --git a/buildscripts/resmokelib/testing/hooks/validate_background.py b/buildscripts/resmokelib/testing/hooks/validate_background.py new file mode 100644 index 00000000000..239b4f77d49 --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/validate_background.py @@ -0,0 +1,81 @@ +"""Test hook for running background validate collection commands against a standalone or members of a replica. + +This hook runs continously in a background thread while the test is running. + +This should not be run against a sharded cluster! +""" + +import os.path + +from buildscripts.resmokelib import errors +from buildscripts.resmokelib.testing.hooks import jsfile +from buildscripts.resmokelib.testing.testcases import interface as testcase +from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase + + +class ValidateCollectionsInBackground(jsfile.JSHook): + """A hook to run background collection validation against test servers while a test is running.""" + + def __init__(self, hook_logger, fixture, shell_options=None): + """Initialize ValidateCollectionsInBackground.""" + description = "Run background collection validation against all mongods while a test is running" + js_filename = os.path.join("jstests", "hooks", "run_validate_collections_background.js") + jsfile.JSHook.__init__(self, hook_logger, fixture, js_filename, description, + shell_options=shell_options) + + self._background_job = None + + def before_suite(self, test_report): + """Start the background thread.""" + server_status = self.fixture.mongo_client().admin.command("serverStatus") + if not server_status["storageEngine"].get("supportsCheckpointCursors", False): + self.logger.info( + "Not enabling the background collection validation thread because '%s' storage" + " engine does not support checkpoints.", server_status["storageEngine"]["name"]) + return + + self._background_job = _BackgroundJob("ValidateCollectionsInBackground") + self.logger.info("Starting the background collection validation thread.") + self._background_job.start() + + def after_suite(self, test_report): + """Signal the background collection validation thread to exit, and wait until it does.""" + if self._background_job is None: + return + + self.logger.info("Stopping the background collection validation thread.") + self._background_job.stop() + + def before_test(self, test, test_report): + """Instruct the background collection validation thread to run while 'test' is also running.""" + if self._background_job is None: + return + + hook_test_case = _ContinuousDynamicJSTestCase.create_before_test( + self.logger.test_case_logger, test, self, self._js_filename, self._shell_options) + hook_test_case.configure(self.fixture) + + self.logger.info("Resuming the background collection validation thread.") + self._background_job.resume(hook_test_case, test_report) + + def after_test(self, test, test_report): # noqa: D205,D400 + """Instruct the background collection validation thread to stop running now that 'test' has + finished running. + """ + if self._background_job is None: + return + + self.logger.info("Pausing the background collection validation thread.") + self._background_job.pause() + + if self._background_job.exc_info is not None: + if isinstance(self._background_job.exc_info[1], errors.TestFailure): + # If the mongo shell process running the JavaScript file exited with a non-zero + # return code, then we raise an errors.ServerFailure exception to cause resmoke.py's + # test execution to stop. + raise errors.ServerFailure(self._background_job.exc_info[1].args[0]) + else: + self.logger.error( + "Encountered an error inside the background collection validation thread.", + exc_info=self._background_job.exc_info) + raise self._background_job.exc_info[1] |