SERVER-56308 Fix auth errors in tenant migration stepdown/kill/terminate suites

(cherry picked from commit ecb77d4c6fc228ac7a4e0b9e385b63eb3919756b)

10 files changed, 44 insertions, 52 deletions

diff --git a/.resmoke_start_time.yml b/.resmoke_start_time.yml
new file mode 100644
index 00000000000..2ceae26189b
--- /dev/null
+++ b/.resmoke_start_time.yml
@@ -0,0 +1 @@
+start_time: 1619557172.2219117
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml
index 1a1a7cb9b38..9ecd6ab76dc 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml
@@ -237,6 +237,7 @@ executor:
   hooks:
   - class: ContinuousStepdown
     kill: true
+    auth_options: *authOptions
   - class: ContinuousTenantMigration
     shell_options:
       global_vars:
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml
index e6d8b3930b2..0629f4e3751 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml
@@ -232,6 +232,7 @@ executor:
       readMode: commands
   hooks:
   - class: ContinuousStepdown
+    auth_options: *authOptions
   - class: ContinuousTenantMigration
     shell_options:
       global_vars:
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml
index 1c02bc8d8ef..7aa73bac471 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml
@@ -233,11 +233,13 @@ executor:
   hooks:
   - class: ContinuousStepdown
     terminate: true
+    auth_options: *authOptions
   - class: ContinuousTenantMigration
     shell_options:
       global_vars:
-        TestData: *TestData
-        authOptions: *authOptions
+        TestData:
+          <<: *TestData
+          authOptions: *authOptions
   # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
   # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
   # validating the entire contents of the collection.
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index 0db4716f868..94dd3617c9c 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -339,5 +339,14 @@ def create_fixture_table(fixture):
     return "Fixture status:\n" + table
 
 
+def authenticate(client, auth_options=None):
+    """Authenticate client for the 'authenticationDatabase' and return the client."""
+    if auth_options is not None:
+        auth_db = client[auth_options["authenticationDatabase"]]
+        auth_db.authenticate(auth_options["username"], password=auth_options["password"],
+                             mechanism=auth_options["authenticationMechanism"])
+    return client
+
+
 # Represents a row in a node info table.
 NodeInfo = namedtuple('NodeInfo', ['full_name', 'name', 'port', 'pid'])
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 3ffc198ff66..5ed92324aee 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -181,8 +181,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
 
         repl_config = {"_id": self.replset_name, "protocolVersion": 1}
         client = self.nodes[0].mongo_client()
-
-        self.auth(client, self.auth_options)
+        interface.authenticate(client, self.auth_options)
 
         if client.local.system.replset.count():
             # Skip initializing the replset if there is an existing configuration.
@@ -318,7 +317,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
     def await_last_op_committed(self):
         """Wait for the last majority committed op to be visible."""
         primary_client = self.get_primary().mongo_client()
-        self.auth(primary_client, self.auth_options)
+        interface.authenticate(primary_client, self.auth_options)
 
         primary_optime = replicaset_utils.get_last_optime(primary_client)
         up_to_date_nodes = set()
@@ -376,16 +375,6 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
                 time.sleep(0.1)  # Wait a little bit before trying again.
             self.logger.info("Secondary on port %d is now available.", secondary.port)
 
-    @staticmethod
-    def auth(client, auth_options=None):
-        """Auth a client connection."""
-        if auth_options is not None:
-            auth_db = client[auth_options["authenticationDatabase"]]
-            auth_db.authenticate(auth_options["username"], password=auth_options["password"],
-                                 mechanism=auth_options["authenticationMechanism"])
-
-        return client
-
     def _await_stable_recovery_timestamp(self):
         """
         Awaits stable recovery timestamps on all nodes in the replica set.
@@ -403,7 +392,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         # Since this method is called at startup we expect the first node to be primary even when
         # self.all_nodes_electable is True.
         primary_client = self.nodes[0].mongo_client()
-        self.auth(primary_client, self.auth_options)
+        interface.authenticate(primary_client, self.auth_options)
 
         # All nodes must be in primary/secondary state prior to this point. Perform a majority
         # write to ensure there is a committed operation on the set. The commit point will
@@ -417,7 +406,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
             self.logger.info("Waiting for node on port %d to have a stable recovery timestamp.",
                              node.port)
             client = node.mongo_client(read_preference=pymongo.ReadPreference.SECONDARY)
-            self.auth(client, self.auth_options)
+            interface.authenticate(client, self.auth_options)
 
             client_admin = client["admin"]
 
@@ -475,7 +464,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         self.logger.info("Waiting to remove all 'newlyAdded' fields")
         primary = self.get_primary()
         client = primary.mongo_client()
-        self.auth(client, self.auth_options)
+        interface.authenticate(client, self.auth_options)
         while self._should_await_newly_added_removals_longer(client):
             time.sleep(0.1)  # Wait a little bit before trying again.
         self.logger.info("All 'newlyAdded' fields removed")
@@ -563,7 +552,8 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
 
                 try:
                     if node.port not in clients:
-                        clients[node.port] = self.auth(node.mongo_client(), self.auth_options)
+                        clients[node.port] = interface.authenticate(node.mongo_client(),
+                                                                    self.auth_options)
 
                     if fn(clients[node.port], node):
                         return node
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 35e49232be4..8cd5dfcdf18 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -146,7 +146,7 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
             mongos.await_ready()
 
         client = self.mongo_client()
-        self._auth_to_db(client)
+        interface.authenticate(client, self.auth_options)
 
         # Turn off the balancer if it is not meant to be enabled.
         if not self.enable_balancer:
@@ -196,7 +196,7 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
                               for mongod in shard.nodes]
 
             for client, port in mongod_clients:
-                self._auth_to_db(client)
+                interface.authenticate(client, self.auth_options)
 
                 while True:
                     # The choice of namespace (local.fooCollection) does not affect the output.
@@ -211,25 +211,17 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
                             .format(port, standalone.MongoDFixture.AWAIT_READY_TIMEOUT_SECS))
                     time.sleep(0.1)
 
-    def _auth_to_db(self, client):
-        """Authenticate client for the 'authenticationDatabase'."""
-        if self.auth_options is not None:
-            auth_db = client[self.auth_options["authenticationDatabase"]]
-            auth_db.authenticate(self.auth_options["username"],
-                                 password=self.auth_options["password"],
-                                 mechanism=self.auth_options["authenticationMechanism"])
-
     def stop_balancer(self, timeout_ms=60000):
         """Stop the balancer."""
         client = self.mongo_client()
-        self._auth_to_db(client)
+        interface.authenticate(client, self.auth_options)
         client.admin.command({"balancerStop": 1}, maxTimeMS=timeout_ms)
         self.logger.info("Stopped the balancer")
 
     def start_balancer(self, timeout_ms=60000):
         """Start the balancer."""
         client = self.mongo_client()
-        self._auth_to_db(client)
+        interface.authenticate(client, self.auth_options)
         client.admin.command({"balancerStart": 1}, maxTimeMS=timeout_ms)
         self.logger.info("Started the balancer")
 
diff --git a/buildscripts/resmokelib/testing/fixtures/tenant_migration.py b/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
index 5fbd20440fe..50e2375868f 100644
--- a/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
+++ b/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
@@ -157,20 +157,10 @@ class TenantMigrationFixture(interface.Fixture):  # pylint: disable=too-many-ins
             output += replica_set.get_node_info()
         return output
 
-    @staticmethod
-    def auth(client, auth_options=None):
-        """Auth a client connection."""
-        if auth_options is not None:
-            auth_db = client[auth_options["authenticationDatabase"]]
-            auth_db.authenticate(auth_options["username"], password=auth_options["password"],
-                                 mechanism=auth_options["authenticationMechanism"])
-
-        return client
-
     def _create_tenant_migration_donor_and_recipient_roles(self, rs):
         """Create a role for tenant migration donor and recipient."""
         primary = rs.get_primary()
-        primary_client = self.auth(primary.mongo_client(), self.auth_options)
+        primary_client = interface.authenticate(primary.mongo_client(), self.auth_options)
 
         try:
             primary_client.admin.command({
diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py
index ec6b4dff0aa..3a85b8e95fa 100644
--- a/buildscripts/resmokelib/testing/hooks/stepdown.py
+++ b/buildscripts/resmokelib/testing/hooks/stepdown.py
@@ -28,7 +28,7 @@ class ContinuousStepdown(interface.Hook):  # pylint: disable=too-many-instance-a
             self, hook_logger, fixture, config_stepdown=True, shard_stepdown=True,
             stepdown_interval_ms=8000, terminate=False, kill=False,
             use_stepdown_permitted_file=False, wait_for_mongos_retarget=False,
-            stepdown_via_heartbeats=True, background_reconfig=False):
+            stepdown_via_heartbeats=True, background_reconfig=False, auth_options=None):
         """Initialize the ContinuousStepdown.
 
         Args:
@@ -66,6 +66,7 @@ class ContinuousStepdown(interface.Hook):  # pylint: disable=too-many-instance-a
         self._kill = kill
 
         self._background_reconfig = background_reconfig
+        self._auth_options = auth_options
 
         # The stepdown file names need to match the same construction as found in
         # jstests/concurrency/fsm_libs/resmoke_runner.js.
@@ -90,7 +91,8 @@ class ContinuousStepdown(interface.Hook):  # pylint: disable=too-many-instance-a
         self._stepdown_thread = _StepdownThread(
             self.logger, self._mongos_fixtures, self._rs_fixtures, self._stepdown_interval_secs,
             self._terminate, self._kill, lifecycle, self._wait_for_mongos_retarget,
-            self._stepdown_via_heartbeats, self._background_reconfig, self._fixture)
+            self._stepdown_via_heartbeats, self._background_reconfig, self._fixture,
+            self._auth_options)
         self.logger.info("Starting the stepdown thread.")
         self._stepdown_thread.start()
 
@@ -352,7 +354,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
     def __init__(  # pylint: disable=too-many-arguments
             self, logger, mongos_fixtures, rs_fixtures, stepdown_interval_secs, terminate, kill,
             stepdown_lifecycle, wait_for_mongos_retarget, stepdown_via_heartbeats,
-            background_reconfig, fixture):
+            background_reconfig, fixture, auth_options=None):
         """Initialize _StepdownThread."""
         threading.Thread.__init__(self, name="StepdownThread")
         self.daemon = True
@@ -371,6 +373,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
         self._stepdown_via_heartbeats = stepdown_via_heartbeats
         self._background_reconfig = background_reconfig
         self._fixture = fixture
+        self._auth_options = auth_options
 
         self._last_exec = time.time()
         # Event set when the thread has been stopped using the 'stop()' method.
@@ -480,6 +483,9 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
         for fixture in self._rs_fixtures:
             fixture.get_primary()
 
+    def _create_client(self, node):
+        return fixture_interface.authenticate(node.mongo_client(), self._auth_options)
+
     def _step_down_all(self):
         for rs_fixture in self._rs_fixtures:
             self._step_down(rs_fixture)
@@ -533,7 +539,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
             self.logger.info("Stepping down the primary on port %d of replica set '%s'.",
                              primary.port, rs_fixture.replset_name)
             try:
-                client = primary.mongo_client()
+                client = self._create_client(primary)
                 client.admin.command(
                     bson.SON([
                         ("replSetStepDown", self._stepdown_duration_secs),
@@ -565,7 +571,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
                              chosen.port, rs_fixture.replset_name)
 
             try:
-                client = chosen.mongo_client()
+                client = self._create_client(chosen)
                 client.admin.command("replSetStepUp")
                 break
             except pymongo.errors.OperationFailure:
@@ -607,7 +613,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
             # introduced a stepdown period on the former primary and so we have to run the
             # {replSetFreeze: 0} command to ensure the former primary is electable in the next
             # round of _step_down().
-            client = primary.mongo_client()
+            client = self._create_client(primary)
             client.admin.command({"replSetFreeze": 0})
         elif secondaries:
             # We successfully stepped up a secondary, wait for the former primary to step down via
@@ -619,7 +625,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
                 chosen.port, rs_fixture.replset_name)
             while True:
                 try:
-                    client = primary.mongo_client()
+                    client = self._create_client(primary)
                     is_secondary = client.admin.command("isMaster")["secondary"]
                     if is_secondary:
                         break
@@ -641,7 +647,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
             retry_start_time = time.time()
             while True:
                 try:
-                    client = primary.mongo_client()
+                    client = self._create_client(primary)
                     client.admin.command("replSetStepUp")
                     break
                 except pymongo.errors.OperationFailure:
@@ -670,7 +676,7 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
         for mongos_fixture in self._mongos_fixtures:
             mongos_conn_str = mongos_fixture.get_internal_connection_string()
             try:
-                client = mongos_fixture.mongo_client()
+                client = self._create_client(mongos_fixture)
             except pymongo.errors.AutoReconnect:
                 pass
             for db in client.database_names():
diff --git a/buildscripts/resmokelib/testing/hooks/tenant_migration.py b/buildscripts/resmokelib/testing/hooks/tenant_migration.py
index f5ba6055b0c..f175c468934 100644
--- a/buildscripts/resmokelib/testing/hooks/tenant_migration.py
+++ b/buildscripts/resmokelib/testing/hooks/tenant_migration.py
@@ -362,7 +362,7 @@ class _TenantMigrationThread(threading.Thread):  # pylint: disable=too-many-inst
         return _TenantMigrationOptions(donor_rs, recipient_rs, self._tenant_id, read_preference)
 
     def _create_client(self, node):
-        return self._tenant_migration_fixture.auth(node.mongo_client(), self._auth_options)
+        return fixture_interface.authenticate(node.mongo_client(), self._auth_options)
 
     def _check_tenant_migration_dbhash(self, migration_opts):
         # Set the donor connection string, recipient connection string, and migration uuid string