SERVER-48178 Finding self in reconfig may be interrupted by closing connections due to rollback

10 files changed, 50 insertions, 20 deletions

diff --git a/src/mongo/client/async_client.cpp b/src/mongo/client/async_client.cpp
index 26a916390de..55cf701b484 100644
--- a/src/mongo/client/async_client.cpp
+++ b/src/mongo/client/async_client.cpp
@@ -174,7 +174,10 @@ Future<void> AsyncDBClient::authenticateInternal(boost::optional<std::string> me
     }
 #endif
 
-    return auth::authenticateInternalClient(clientName, mechanismHint, _makeAuthRunCommandHook());
+    return auth::authenticateInternalClient(clientName,
+                                            mechanismHint,
+                                            auth::StepDownBehavior::kKillConnection,
+                                            _makeAuthRunCommandHook());
 }
 
 Future<bool> AsyncDBClient::completeSpeculativeAuth(std::shared_ptr<SaslClientSession> session,
diff --git a/src/mongo/client/authenticate.cpp b/src/mongo/client/authenticate.cpp
index a69054ba781..839aef7462e 100644
--- a/src/mongo/client/authenticate.cpp
+++ b/src/mongo/client/authenticate.cpp
@@ -250,13 +250,20 @@ BSONObj getInternalAuthParams(size_t idx, const std::string& mechanism) {
 
 Future<std::string> negotiateSaslMechanism(RunCommandHook runCommand,
                                            const UserName& username,
-                                           boost::optional<std::string> mechanismHint) {
+                                           boost::optional<std::string> mechanismHint,
+                                           StepDownBehavior stepDownBehavior) {
     if (mechanismHint && !mechanismHint->empty()) {
         return Future<std::string>::makeReady(*mechanismHint);
     }
 
-    const auto request =
-        BSON("ismaster" << 1 << "saslSupportedMechs" << username.getUnambiguousName());
+    BSONObjBuilder builder;
+    builder.append("ismaster", 1);
+    builder.append("saslSupportedMechs", username.getUnambiguousName());
+    if (stepDownBehavior == StepDownBehavior::kKeepConnectionOpen) {
+        builder.append("hangUpOnStepDown", false);
+    }
+    const auto request = builder.obj();
+
     return runCommand(OpMsgRequest::fromDBAndBody("admin"_sd, std::move(request)))
         .then([](BSONObj reply) -> Future<std::string> {
             auto mechsArrayObj = reply.getField("saslSupportedMechs");
@@ -285,8 +292,10 @@ Future<std::string> negotiateSaslMechanism(RunCommandHook runCommand,
 
 Future<void> authenticateInternalClient(const std::string& clientSubjectName,
                                         boost::optional<std::string> mechanismHint,
+                                        StepDownBehavior stepDownBehavior,
                                         RunCommandHook runCommand) {
-    return negotiateSaslMechanism(runCommand, internalSecurity.user->getName(), mechanismHint)
+    return negotiateSaslMechanism(
+               runCommand, internalSecurity.user->getName(), mechanismHint, stepDownBehavior)
         .then([runCommand, clientSubjectName](std::string mechanism) -> Future<void> {
             auto params = getInternalAuthParams(0, mechanism);
             if (params.isEmpty()) {
diff --git a/src/mongo/client/authenticate.h b/src/mongo/client/authenticate.h
index 326b3c1fc5d..74b858f237c 100644
--- a/src/mongo/client/authenticate.h
+++ b/src/mongo/client/authenticate.h
@@ -74,6 +74,11 @@ constexpr auto kSpeculativeAuthenticate = "speculativeAuthenticate"_sd;
 constexpr auto kAuthenticateCommand = "authenticate"_sd;
 
 /**
+ * On replication step down, should the current connection be killed or left open.
+ */
+enum class StepDownBehavior { kKillConnection, kKeepConnectionOpen };
+
+/**
  * Authenticate a user.
  *
  * Pass the default hostname for this client in through "hostname." If SSL is enabled and
@@ -113,11 +118,15 @@ Future<void> authenticateClient(const BSONObj& params,
  * (e.g. SCRAM-SHA-256). If it is boost::none, then an isMaster will be called to negotiate
  * a SASL mechanism with the server.
  *
+ * The "stepDownBehavior" parameter controls whether replication will kill the connection on
+ * stepdown.
+ *
  * Because this may retry during cluster keyfile rollover, this may call the RunCommandHook more
  * than once, but will only call the AuthCompletionHandler once.
  */
 Future<void> authenticateInternalClient(const std::string& clientSubjectName,
                                         boost::optional<std::string> mechanismHint,
+                                        StepDownBehavior stepDownBehavior,
                                         RunCommandHook runCommand);
 
 /**
@@ -162,7 +171,8 @@ BSONObj buildAuthParams(StringData dbname,
  */
 Future<std::string> negotiateSaslMechanism(RunCommandHook runCommand,
                                            const UserName& username,
-                                           boost::optional<std::string> mechanismHint);
+                                           boost::optional<std::string> mechanismHint,
+                                           StepDownBehavior stepDownBehavior);
 
 /**
  * Return the field name for the database containing credential information.
diff --git a/src/mongo/client/dbclient_base.cpp b/src/mongo/client/dbclient_base.cpp
index f3112fb7a59..f13eb6a1b3c 100644
--- a/src/mongo/client/dbclient_base.cpp
+++ b/src/mongo/client/dbclient_base.cpp
@@ -475,7 +475,7 @@ void DBClientBase::_auth(const BSONObj& params) {
         .get();
 }
 
-Status DBClientBase::authenticateInternalUser() {
+Status DBClientBase::authenticateInternalUser(auth::StepDownBehavior stepDownBehavior) {
     ScopedMetadataWriterRemover remover{this};
     if (!auth::isInternalAuthSet()) {
         if (!serverGlobalParams.quiet.load()) {
@@ -493,9 +493,9 @@ Status DBClientBase::authenticateInternalUser() {
     }
 #endif
 
-    auto status =
-        auth::authenticateInternalClient(clientName, boost::none, _makeAuthRunCommandHook())
-            .getNoThrow();
+    auto status = auth::authenticateInternalClient(
+                      clientName, boost::none, stepDownBehavior, _makeAuthRunCommandHook())
+                      .getNoThrow();
     if (status.isOK()) {
         return status;
     }
diff --git a/src/mongo/client/dbclient_base.h b/src/mongo/client/dbclient_base.h
index 868af85b763..4da50dcb286 100644
--- a/src/mongo/client/dbclient_base.h
+++ b/src/mongo/client/dbclient_base.h
@@ -304,7 +304,8 @@ public:
      * Authenticates to another cluster member using appropriate authentication data.
      * @return true if the authentication was successful
      */
-    virtual Status authenticateInternalUser();
+    virtual Status authenticateInternalUser(
+        auth::StepDownBehavior stepDownBehavior = auth::StepDownBehavior::kKillConnection);
 
     /**
      * Authenticate a user.
diff --git a/src/mongo/client/dbclient_connection.cpp b/src/mongo/client/dbclient_connection.cpp
index 8a3eeadc8f9..4b24f0652d2 100644
--- a/src/mongo/client/dbclient_connection.cpp
+++ b/src/mongo/client/dbclient_connection.cpp
@@ -257,12 +257,13 @@ void DBClientConnection::_auth(const BSONObj& params) {
     DBClientBase::_auth(params);
 }
 
-Status DBClientConnection::authenticateInternalUser() {
+Status DBClientConnection::authenticateInternalUser(auth::StepDownBehavior stepDownBehavior) {
     if (autoReconnect) {
         _internalAuthOnReconnect = true;
+        _internalAuthStepDownBehavior = stepDownBehavior;
     }
 
-    return DBClientBase::authenticateInternalUser();
+    return DBClientBase::authenticateInternalUser(stepDownBehavior);
 }
 
 bool DBClientConnection::connect(const HostAndPort& server,
@@ -586,7 +587,7 @@ void DBClientConnection::_checkConnection() {
                 "Reconnected",
                 "connString"_attr = toString());
     if (_internalAuthOnReconnect) {
-        uassertStatusOK(authenticateInternalUser());
+        uassertStatusOK(authenticateInternalUser(_internalAuthStepDownBehavior));
     } else {
         for (const auto& kv : authCache) {
             try {
diff --git a/src/mongo/client/dbclient_connection.h b/src/mongo/client/dbclient_connection.h
index 79cbbc1ed59..340c8670b3d 100644
--- a/src/mongo/client/dbclient_connection.h
+++ b/src/mongo/client/dbclient_connection.h
@@ -300,7 +300,8 @@ public:
         return _isMongos;
     }
 
-    Status authenticateInternalUser() override;
+    Status authenticateInternalUser(
+        auth::StepDownBehavior stepDownBehavior = auth::StepDownBehavior::kKillConnection) override;
 
     bool authenticatedDuringConnect() const override {
         return _authenticatedDuringConnect;
@@ -338,6 +339,9 @@ protected:
     void _checkConnection();
 
     bool _internalAuthOnReconnect = false;
+
+    auth::StepDownBehavior _internalAuthStepDownBehavior = auth::StepDownBehavior::kKillConnection;
+
     std::map<std::string, BSONObj> authCache;
 
     static AtomicWord<int> _numConnections;
diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp
index e10bd070301..68ac0b84b50 100644
--- a/src/mongo/client/dbclient_rs.cpp
+++ b/src/mongo/client/dbclient_rs.cpp
@@ -482,15 +482,16 @@ Status DBClientReplicaSet::_runAuthLoop(Authenticate authCb) {
     }
 }
 
-Status DBClientReplicaSet::authenticateInternalUser() {
+Status DBClientReplicaSet::authenticateInternalUser(auth::StepDownBehavior stepDownBehavior) {
     if (!auth::isInternalAuthSet()) {
         return {ErrorCodes::AuthenticationFailed,
                 "No authentication parameters set for internal user"};
     }
 
     _internalAuthRequested = true;
-    return _runAuthLoop(
-        [&](DBClientConnection* conn) { uassertStatusOK(conn->authenticateInternalUser()); });
+    return _runAuthLoop([stepDownBehavior](DBClientConnection* conn) {
+        uassertStatusOK(conn->authenticateInternalUser(stepDownBehavior));
+    });
 }
 
 void DBClientReplicaSet::_auth(const BSONObj& params) {
diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h
index 9603842f7d9..4069576cba0 100644
--- a/src/mongo/client/dbclient_rs.h
+++ b/src/mongo/client/dbclient_rs.h
@@ -75,7 +75,7 @@ public:
      */
     bool connect();
 
-    Status authenticateInternalUser() override;
+    Status authenticateInternalUser(auth::StepDownBehavior stepDownBehavior) override;
 
     /**
      * Logs out the connection for the given database.
diff --git a/src/mongo/db/repl/isself.cpp b/src/mongo/db/repl/isself.cpp
index ea556923a4b..4e6a544dd5b 100644
--- a/src/mongo/db/repl/isself.cpp
+++ b/src/mongo/db/repl/isself.cpp
@@ -236,7 +236,8 @@ bool isSelf(const HostAndPort& hostAndPort, ServiceContext* const ctx) {
         }
 
         if (auth::isInternalAuthSet()) {
-            auto authInternalUserResult = conn.authenticateInternalUser();
+            auto authInternalUserResult =
+                conn.authenticateInternalUser(auth::StepDownBehavior::kKeepConnectionOpen);
             if (!authInternalUserResult.isOK()) {
                 LOGV2(4834701,
                       "isSelf could not authenticate internal user",