SERVER-60779 Extend CheckReplDBHash to work with TenantMigrationsFixture and ClusterToClusterFixture

7 files changed, 65 insertions, 17 deletions

diff --git a/buildscripts/resmokelib/testing/fixtures/cluster_to_cluster.py b/buildscripts/resmokelib/testing/fixtures/cluster_to_cluster.py
index 1c5f0540ded..1a9d45ce7d7 100644
--- a/buildscripts/resmokelib/testing/fixtures/cluster_to_cluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/cluster_to_cluster.py
@@ -6,7 +6,7 @@ import os.path
 import buildscripts.resmokelib.testing.fixtures.interface as interface
 
 
-class ClusterToClusterFixture(interface.Fixture):  # pylint: disable=too-many-instance-attributes
+class ClusterToClusterFixture(interface.MultiClusterFixture):  # pylint: disable=too-many-instance-attributes
     """Fixture which provides two clusters to perform a cluster to cluster replication."""
 
     def __init__(  # pylint: disable=too-many-arguments,too-many-locals
@@ -14,7 +14,8 @@ class ClusterToClusterFixture(interface.Fixture):  # pylint: disable=too-many-in
             dbpath_prefix=None, preserve_dbpath=False):
         """Initialize with different options for the clusters."""
 
-        interface.Fixture.__init__(self, logger, job_num, fixturelib, dbpath_prefix=dbpath_prefix)
+        interface.MultiClusterFixture.__init__(self, logger, job_num, fixturelib,
+                                               dbpath_prefix=dbpath_prefix)
 
         self.clusters = []
         self.both_cluster_options = []
@@ -56,6 +57,9 @@ class ClusterToClusterFixture(interface.Fixture):  # pylint: disable=too-many-in
 
         self.both_cluster_options = parsed_options
 
+        # The cluster that starts off with the data.
+        self.source_cluster_index = 0
+
     def setup(self):
         """Set up the cluster to cluster fixture according to the options provided."""
 
@@ -126,14 +130,10 @@ class ClusterToClusterFixture(interface.Fixture):  # pylint: disable=too-many-in
             raise ValueError("Must call setup() before calling get_internal_connection_string")
         return self.clusters[0].get_internal_connection_string()
 
-    def get_cluster0_connection_string(self):
-        """Return the connection string of cluster 0."""
-        if not self.clusters:
-            raise ValueError("Must call setup() before calling get_cluster0_connection_string")
-        return self.clusters[0].get_internal_connection_string()
+    def get_independent_clusters(self):
+        """Return the clusters involved in cluster to cluster replication."""
+        return self.clusters.copy()
 
-    def get_cluster1_connection_string(self):
-        """Return the connection string of cluster 1."""
-        if not self.clusters:
-            raise ValueError("Must call setup() before calling get_cluster1_connection_string")
-        return self.clusters[1].get_internal_connection_string()
+    def reverse_replication_direction(self):
+        """Swap the source and destination clusters."""
+        self.source_cluster_index = 1 - self.source_cluster_index
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index 1e11f4cb356..97e22902c02 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -206,6 +206,23 @@ class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):  #
         return "%r(%r, %r)" % (self.__class__.__name__, self.logger, self.job_num)
 
 
+class MultiClusterFixture(Fixture):
+    """
+    Base class for fixtures that may consist of multiple independent participant clusters.
+
+    The participant clusters can function independently without coordination, but are bound together
+    only for some duration as they participate in some process such as a migration. The participant
+    clusters are fixtures themselves.
+    """
+
+    REGISTERED_NAME = registry.LEAVE_UNREGISTERED  # type: ignore
+
+    def get_independent_clusters(self):
+        """Return a list of the independent clusters (fixtures) that participate in this fixture."""
+        raise NotImplementedError(
+            "get_independent_clusters must be implemented by MultiClusterFixture subclasses")
+
+
 class ReplFixture(Fixture):
     """Base class for all fixtures that support replication."""
 
diff --git a/buildscripts/resmokelib/testing/fixtures/tenant_migration.py b/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
index 9431c2116b4..3b1810e6a57 100644
--- a/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
+++ b/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
@@ -6,7 +6,7 @@ import buildscripts.resmokelib.testing.fixtures.interface as interface
 from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
 
 
-class TenantMigrationFixture(interface.Fixture):  # pylint: disable=too-many-instance-attributes
+class TenantMigrationFixture(interface.MultiClusterFixture):  # pylint: disable=too-many-instance-attributes
     """Fixture which provides JSTests with a set of replica sets to run tenant migration against."""
 
     def __init__(  # pylint: disable=too-many-arguments,too-many-locals
@@ -19,7 +19,8 @@ class TenantMigrationFixture(interface.Fixture):  # pylint: disable=too-many-ins
             default_read_concern=None, default_write_concern=None):
         """Initialize TenantMigrationFixture with different options for the replica set processes."""
 
-        interface.Fixture.__init__(self, logger, job_num, fixturelib, dbpath_prefix=dbpath_prefix)
+        interface.MultiClusterFixture.__init__(self, logger, job_num, fixturelib,
+                                               dbpath_prefix=dbpath_prefix)
 
         self.common_mongod_options = self.fixturelib.default_if_none(common_mongod_options, {})
         self.per_mongod_options = self.fixturelib.default_if_none(per_mongod_options, {})
@@ -151,6 +152,10 @@ class TenantMigrationFixture(interface.Fixture):  # pylint: disable=too-many-ins
             output += replica_set.get_node_info()
         return output
 
+    def get_independent_clusters(self):
+        """Return the replica sets involved in the tenant migration."""
+        return self.replica_sets.copy()
+
     def _create_tenant_migration_donor_and_recipient_roles(self, rs):
         """Create a role for tenant migration donor and recipient."""
         primary = rs.get_primary()
diff --git a/buildscripts/resmokelib/testing/hooks/dbhash.py b/buildscripts/resmokelib/testing/hooks/dbhash.py
index c3e20d5f141..9f71fc1aeb6 100644
--- a/buildscripts/resmokelib/testing/hooks/dbhash.py
+++ b/buildscripts/resmokelib/testing/hooks/dbhash.py
@@ -5,7 +5,7 @@ import os.path
 from buildscripts.resmokelib.testing.hooks import jsfile
 
 
-class CheckReplDBHash(jsfile.DataConsistencyHook):
+class CheckReplDBHash(jsfile.PerClusterDataConsistencyHook):
     """Check if the dbhashes match.
 
     This includes dbhashes for all non-local databases and non-replicated system collections that
diff --git a/buildscripts/resmokelib/testing/hooks/jsfile.py b/buildscripts/resmokelib/testing/hooks/jsfile.py
index 1075cb26ac4..f9e02b416f9 100644
--- a/buildscripts/resmokelib/testing/hooks/jsfile.py
+++ b/buildscripts/resmokelib/testing/hooks/jsfile.py
@@ -2,6 +2,7 @@
 
 from buildscripts.resmokelib import errors
 from buildscripts.resmokelib.testing.hooks import interface
+from buildscripts.resmokelib.testing.fixtures.interface import MultiClusterFixture
 from buildscripts.resmokelib.testing.testcases import jstest
 from buildscripts.resmokelib.utils import registry
 
@@ -55,6 +56,31 @@ class DataConsistencyHook(JSHook):
             raise errors.ServerFailure(err.args[0])
 
 
+class PerClusterDataConsistencyHook(DataConsistencyHook):
+    """
+    A hook that runs on each independent cluster of the fixture.
+
+    The independent cluster itself may be another fixture.
+    """
+
+    REGISTERED_NAME = registry.LEAVE_UNREGISTERED
+
+    def after_test(self, test, test_report):
+        """After test execution."""
+
+        # Break the fixture down into its participant clusters if it is a MultiClusterFixture.
+        clusters = [self.fixture] if not isinstance(self.fixture, MultiClusterFixture)\
+                    else self.fixture.get_independent_clusters()
+
+        for cluster in clusters:
+            self.logger.info("Running jsfile '%s' on '%s' with driver URL '%s'", self._js_filename,
+                             cluster, cluster.get_driver_connection_url())
+            hook_test_case = DynamicJSTestCase.create_after_test(
+                test.logger, test, self, self._js_filename, self._shell_options)
+            hook_test_case.configure(self.fixture)
+            hook_test_case.run_dynamic_test(test_report)
+
+
 class DynamicJSTestCase(interface.DynamicTestCase):
     """A dynamic TestCase that runs a JavaScript file."""
 
diff --git a/buildscripts/resmokelib/testing/hooks/oplog.py b/buildscripts/resmokelib/testing/hooks/oplog.py
index 0dc0bc79f88..d66352180da 100644
--- a/buildscripts/resmokelib/testing/hooks/oplog.py
+++ b/buildscripts/resmokelib/testing/hooks/oplog.py
@@ -5,7 +5,7 @@ import os.path
 from buildscripts.resmokelib.testing.hooks import jsfile
 
 
-class CheckReplOplogs(jsfile.DataConsistencyHook):  # pylint: disable=non-parent-init-called,super-init-not-called
+class CheckReplOplogs(jsfile.PerClusterDataConsistencyHook):  # pylint: disable=non-parent-init-called,super-init-not-called
     """Check that local.oplog.rs matches on the primary and secondaries."""
 
     IS_BACKGROUND = False
diff --git a/buildscripts/resmokelib/testing/hooks/validate.py b/buildscripts/resmokelib/testing/hooks/validate.py
index c10b75b8e00..a4417e2d848 100644
--- a/buildscripts/resmokelib/testing/hooks/validate.py
+++ b/buildscripts/resmokelib/testing/hooks/validate.py
@@ -5,7 +5,7 @@ import os.path
 from buildscripts.resmokelib.testing.hooks import jsfile
 
 
-class ValidateCollections(jsfile.DataConsistencyHook):
+class ValidateCollections(jsfile.PerClusterDataConsistencyHook):
     """Run full validation.
 
     This will run on all collections in all databases on every stand-alone