SERVER-42358 Add new background collection validation test hook. Runs concurrently with tests.

12 files changed, 210 insertions, 104 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_replication.yml
index a2270cc4c32..d1500a90d79 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication.yml
@@ -19,6 +19,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - ValidateCollections
     tests: true
@@ -32,6 +33,7 @@ executor:
   #
   # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplDBHash
   - class: ValidateCollections
   - class: CleanupConcurrencyWorkloads
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml
index db949dd4e58..4906056a9b7 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml
@@ -46,6 +46,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - ValidateCollections
     tests: true
@@ -64,6 +65,7 @@ executor:
   #
   # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplDBHash
   - class: ValidateCollections
   - class: CleanupConcurrencyWorkloads
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml
index 0ce9d4506fb..dc847ce498d 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml
@@ -46,6 +46,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - ValidateCollections
     tests: true
@@ -64,6 +65,7 @@ executor:
   #
   # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplDBHash
   - class: ValidateCollections
   - class: CleanEveryN
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml
index d535361c29f..c0d263d26be 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication_ubsan.yml
@@ -18,9 +18,10 @@ selector:
 executor:
   archive:
     hooks:
+      - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - ValidateCollections
-      - CheckReplDBHashInBackground
     tests: true
   config:
     shell_options:
@@ -32,6 +33,7 @@ executor:
   #
   # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplDBHash
   - class: ValidateCollections
   - class: CleanEveryN
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml
index 5ee8dbf0bc9..77d98918c18 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_cursor_sweeps.yml
@@ -19,6 +19,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - ValidateCollections
     tests: true
@@ -30,6 +31,7 @@ executor:
   # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
   # validating the entire contents of the collection.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplDBHash
   - class: ValidateCollections
   - class: CleanupConcurrencyWorkloads
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml
index 191f31de952..89a12841254 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication_wiredtiger_eviction_debug.yml
@@ -19,6 +19,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - ValidateCollections
     tests: true
@@ -30,6 +31,7 @@ executor:
   # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
   # validating the entire contents of the collection.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplDBHash
   - class: ValidateCollections
   - class: CleanupConcurrencyWorkloads
diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml
index fabca31780d..af137e2c3ca 100644
--- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml
@@ -29,6 +29,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - CheckReplOplogs
       - ValidateCollections
@@ -41,6 +42,7 @@ executor:
   # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
   # validating the entire contents of the collection.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplOplogs
   - class: CheckReplDBHash
   - class: ValidateCollections
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml
index e1ed3491d77..dca64af5445 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml
@@ -23,6 +23,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - CheckReplOplogs
       - ValidateCollections
@@ -35,6 +36,7 @@ executor:
   # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
   # validating the entire contents of the collection.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplOplogs
   - class: CheckReplDBHash
   - class: ValidateCollections
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml
index 38b0db0d9a4..00a177cc35e 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_large_txns_format_jscore_passthrough.yml
@@ -17,6 +17,7 @@ executor:
   archive:
     hooks:
       - CheckReplDBHashInBackground
+      - ValidateCollectionsInBackground
       - CheckReplDBHash
       - CheckReplOplogs
       - ValidateCollections
@@ -29,6 +30,7 @@ executor:
   # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
   # validating the entire contents of the collection.
   - class: CheckReplDBHashInBackground
+  - class: ValidateCollectionsInBackground
   - class: CheckReplOplogs
   - class: CheckReplDBHash
   - class: ValidateCollections
diff --git a/buildscripts/resmokelib/testing/hooks/background_job.py b/buildscripts/resmokelib/testing/hooks/background_job.py
new file mode 100644
index 00000000000..39daeffb100
--- /dev/null
+++ b/buildscripts/resmokelib/testing/hooks/background_job.py
@@ -0,0 +1,98 @@
+"""Background job utils. Helps run a JS file repeatedly while a test suite's tests run."""
+
+import sys
+import threading
+
+from buildscripts.resmokelib.testing.hooks import jsfile
+
+
+class _BackgroundJob(threading.Thread):  # pylint: disable=too-many-instance-attributes
+    """A thread for running a JS file while a test is running."""
+
+    def __init__(self, thread_name):
+        """Initialize _BackgroundJob."""
+        threading.Thread.__init__(self, name=thread_name)
+        self.daemon = True
+
+        self._lock = threading.Lock()
+        self._cond = threading.Condition(self._lock)
+
+        self._should_stop = False
+        self._should_resume = False
+        self._is_idle = True
+
+        self._hook_test_case = None
+        self._test_report = None
+        self.exc_info = None
+
+    def run(self):
+        while True:
+            with self._lock:
+                while not self._should_resume:
+                    self._cond.wait()
+
+                # We have the background thread set 'self._should_resume' back to false to ensure
+                # that 'self._hook_test_case.run_dynamic_test()' is only called once before the
+                # resume() method is called again.
+                self._should_resume = False
+
+                if self._should_stop:
+                    break
+
+                # We are about to execute 'self._hook_test_case' so we mark the background thread as
+                # no longer being idle.
+                self._is_idle = False
+
+            try:
+                self._hook_test_case.run_dynamic_test(self._test_report)
+            except:  # pylint: disable=bare-except
+                self.exc_info = sys.exc_info()
+            finally:
+                with self._lock:
+                    self._is_idle = True
+                    self._cond.notify_all()
+
+    def stop(self):
+        """Signal the background thread to exit, and wait until it does."""
+        with self._lock:
+            self._should_stop = True
+        self.resume(hook_test_case=None, test_report=None)
+        self.join()
+
+    def pause(self):  # noqa: D205,D400
+        """Signal the background thread that it should stop executing 'self._hook_test_case', and
+        wait until the current execution has finished.
+        """
+        self._hook_test_case.signal_stop_test()
+        with self._lock:
+            while not self._is_idle:
+                self._cond.wait()
+
+    def resume(self, hook_test_case, test_report):
+        """Instruct the background thread to start executing 'hook_test_case'."""
+        self._hook_test_case = hook_test_case
+        self._test_report = test_report
+        self.exc_info = None
+        with self._lock:
+            self._should_resume = True
+            self._cond.notify_all()
+
+
+class _ContinuousDynamicJSTestCase(jsfile.DynamicJSTestCase):
+    """A dynamic TestCase that runs a JavaScript file repeatedly."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize _ContinuousDynamicJSTestCase."""
+        jsfile.DynamicJSTestCase.__init__(self, *args, **kwargs)
+        self._should_stop = threading.Event()
+
+    def run_test(self):
+        """Execute the test repeatedly."""
+        while not self._should_stop.is_set():
+            jsfile.DynamicJSTestCase.run_test(self)
+
+    def signal_stop_test(self):  # noqa: D205,D400
+        """Indicate to the thread executing the run_test() method that the current execution is the
+        last one. This method returns without waiting for the current execution to finish.
+        """
+        self._should_stop.set()
diff --git a/buildscripts/resmokelib/testing/hooks/dbhash_background.py b/buildscripts/resmokelib/testing/hooks/dbhash_background.py
index 877f8c5ef1b..1faf084c58c 100644
--- a/buildscripts/resmokelib/testing/hooks/dbhash_background.py
+++ b/buildscripts/resmokelib/testing/hooks/dbhash_background.py
@@ -5,12 +5,11 @@ running.
 """
 
 import os.path
-import sys
-import threading
 
 from buildscripts.resmokelib import errors
 from buildscripts.resmokelib.testing.hooks import jsfile
 from buildscripts.resmokelib.testing.testcases import interface as testcase
+from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase
 
 
 class CheckReplDBHashInBackground(jsfile.JSHook):
@@ -35,12 +34,13 @@ class CheckReplDBHashInBackground(jsfile.JSHook):
             server_status = client.admin.command("serverStatus")
             if not server_status["storageEngine"].get("supportsSnapshotReadConcern", False):
                 self.logger.info(
-                    "Not enabling the background thread because '%s' storage engine"
-                    " doesn't support snapshot reads.", server_status["storageEngine"]["name"])
+                    "Not enabling the background check repl dbhash thread because '%s' storage"
+                    " engine doesn't support snapshot reads.",
+                    server_status["storageEngine"]["name"])
                 return
 
-        self._background_job = _BackgroundJob()
-        self.logger.info("Starting the background thread.")
+        self._background_job = _BackgroundJob("CheckReplDBHashInBackground")
+        self.logger.info("Starting the background check repl dbhash thread.")
         self._background_job.start()
 
     def after_suite(self, test_report):
@@ -48,7 +48,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook):
         if self._background_job is None:
             return
 
-        self.logger.info("Stopping the background thread.")
+        self.logger.info("Stopping the background check repl dbhash thread.")
         self._background_job.stop()
 
     def before_test(self, test, test_report):
@@ -60,7 +60,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook):
             self.logger.test_case_logger, test, self, self._js_filename, self._shell_options)
         hook_test_case.configure(self.fixture)
 
-        self.logger.info("Resuming the background thread.")
+        self.logger.info("Resuming the background check repl dbhash thread.")
         self._background_job.resume(hook_test_case, test_report)
 
     def after_test(self, test, test_report):  # noqa: D205,D400
@@ -70,7 +70,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook):
         if self._background_job is None:
             return
 
-        self.logger.info("Pausing the background thread.")
+        self.logger.info("Pausing the background check repl dbhash thread.")
         self._background_job.pause()
 
         if self._background_job.exc_info is not None:
@@ -80,98 +80,7 @@ class CheckReplDBHashInBackground(jsfile.JSHook):
                 # test execution to stop.
                 raise errors.ServerFailure(self._background_job.exc_info[1].args[0])
             else:
-                self.logger.error("Encountered an error inside the background thread.",
-                                  exc_info=self._background_job.exc_info)
+                self.logger.error(
+                    "Encountered an error inside the background check repl dbhash thread.",
+                    exc_info=self._background_job.exc_info)
                 raise self._background_job.exc_info[1]
-
-
-class _BackgroundJob(threading.Thread):  # pylint: disable=too-many-instance-attributes
-    """A thread for running the dbhash check while a test is running."""
-
-    def __init__(self):
-        """Initialize _BackgroundJob."""
-        threading.Thread.__init__(self, name="CheckReplDBHashInBackground")
-        self.daemon = True
-
-        self._lock = threading.Lock()
-        self._cond = threading.Condition(self._lock)
-
-        self._should_stop = False
-        self._should_resume = False
-        self._is_idle = True
-
-        self._hook_test_case = None
-        self._test_report = None
-        self.exc_info = None
-
-    def run(self):
-        while True:
-            with self._lock:
-                while not self._should_resume:
-                    self._cond.wait()
-
-                # We have the background thread set 'self._should_resume' back to false to ensure
-                # that 'self._hook_test_case.run_dynamic_test()' is only called once before the
-                # resume() method is called again.
-                self._should_resume = False
-
-                if self._should_stop:
-                    break
-
-                # We are about to execute 'self._hook_test_case' so we mark the background thread as
-                # no longer being idle.
-                self._is_idle = False
-
-            try:
-                self._hook_test_case.run_dynamic_test(self._test_report)
-            except:  # pylint: disable=bare-except
-                self.exc_info = sys.exc_info()
-            finally:
-                with self._lock:
-                    self._is_idle = True
-                    self._cond.notify_all()
-
-    def stop(self):
-        """Signal the background thread to exit, and wait until it does."""
-        with self._lock:
-            self._should_stop = True
-        self.resume(hook_test_case=None, test_report=None)
-        self.join()
-
-    def pause(self):  # noqa: D205,D400
-        """Signal the background thread that it should stop executing 'self._hook_test_case', and
-        wait until the current execution has finished.
-        """
-        self._hook_test_case.signal_stop_test()
-        with self._lock:
-            while not self._is_idle:
-                self._cond.wait()
-
-    def resume(self, hook_test_case, test_report):
-        """Instruct the background thread to start executing 'hook_test_case'."""
-        self._hook_test_case = hook_test_case
-        self._test_report = test_report
-        self.exc_info = None
-        with self._lock:
-            self._should_resume = True
-            self._cond.notify_all()
-
-
-class _ContinuousDynamicJSTestCase(jsfile.DynamicJSTestCase):
-    """A dynamic TestCase that runs a JavaScript file repeatedly."""
-
-    def __init__(self, *args, **kwargs):
-        """Initialize _ContinuousDynamicJSTestCase."""
-        jsfile.DynamicJSTestCase.__init__(self, *args, **kwargs)
-        self._should_stop = threading.Event()
-
-    def run_test(self):
-        """Execute the test repeatedly."""
-        while not self._should_stop.is_set():
-            jsfile.DynamicJSTestCase.run_test(self)
-
-    def signal_stop_test(self):  # noqa: D205,D400
-        """Indicate to the thread executing the run_test() method that the current execution is the
-        last one. This method returns without waiting for the current execution to finish.
-        """
-        self._should_stop.set()
diff --git a/buildscripts/resmokelib/testing/hooks/validate_background.py b/buildscripts/resmokelib/testing/hooks/validate_background.py
new file mode 100644
index 00000000000..239b4f77d49
--- /dev/null
+++ b/buildscripts/resmokelib/testing/hooks/validate_background.py
@@ -0,0 +1,81 @@
+"""Test hook for running background validate collection commands against a standalone or members of a replica.
+
+This hook runs continously in a background thread while the test is running.
+
+This should not be run against a sharded cluster!
+"""
+
+import os.path
+
+from buildscripts.resmokelib import errors
+from buildscripts.resmokelib.testing.hooks import jsfile
+from buildscripts.resmokelib.testing.testcases import interface as testcase
+from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase
+
+
+class ValidateCollectionsInBackground(jsfile.JSHook):
+    """A hook to run background collection validation against test servers while a test is running."""
+
+    def __init__(self, hook_logger, fixture, shell_options=None):
+        """Initialize ValidateCollectionsInBackground."""
+        description = "Run background collection validation against all mongods while a test is running"
+        js_filename = os.path.join("jstests", "hooks", "run_validate_collections_background.js")
+        jsfile.JSHook.__init__(self, hook_logger, fixture, js_filename, description,
+                               shell_options=shell_options)
+
+        self._background_job = None
+
+    def before_suite(self, test_report):
+        """Start the background thread."""
+        server_status = self.fixture.mongo_client().admin.command("serverStatus")
+        if not server_status["storageEngine"].get("supportsCheckpointCursors", False):
+            self.logger.info(
+                "Not enabling the background collection validation thread because '%s' storage"
+                " engine does not support checkpoints.", server_status["storageEngine"]["name"])
+            return
+
+        self._background_job = _BackgroundJob("ValidateCollectionsInBackground")
+        self.logger.info("Starting the background collection validation thread.")
+        self._background_job.start()
+
+    def after_suite(self, test_report):
+        """Signal the background collection validation thread to exit, and wait until it does."""
+        if self._background_job is None:
+            return
+
+        self.logger.info("Stopping the background collection validation thread.")
+        self._background_job.stop()
+
+    def before_test(self, test, test_report):
+        """Instruct the background collection validation thread to run while 'test' is also running."""
+        if self._background_job is None:
+            return
+
+        hook_test_case = _ContinuousDynamicJSTestCase.create_before_test(
+            self.logger.test_case_logger, test, self, self._js_filename, self._shell_options)
+        hook_test_case.configure(self.fixture)
+
+        self.logger.info("Resuming the background collection validation thread.")
+        self._background_job.resume(hook_test_case, test_report)
+
+    def after_test(self, test, test_report):  # noqa: D205,D400
+        """Instruct the background collection validation thread to stop running now that 'test' has
+        finished running.
+        """
+        if self._background_job is None:
+            return
+
+        self.logger.info("Pausing the background collection validation thread.")
+        self._background_job.pause()
+
+        if self._background_job.exc_info is not None:
+            if isinstance(self._background_job.exc_info[1], errors.TestFailure):
+                # If the mongo shell process running the JavaScript file exited with a non-zero
+                # return code, then we raise an errors.ServerFailure exception to cause resmoke.py's
+                # test execution to stop.
+                raise errors.ServerFailure(self._background_job.exc_info[1].args[0])
+            else:
+                self.logger.error(
+                    "Encountered an error inside the background collection validation thread.",
+                    exc_info=self._background_job.exc_info)
+                raise self._background_job.exc_info[1]