SERVER-66050 findSelfInConfig should attempt fast path for every HostAndPort before trying slow path

(cherry picked from commit 0c0f80372c2adee56c16c106ec2711d39ea6b45f)

10 files changed, 167 insertions, 7 deletions

diff --git a/src/mongo/db/repl/isself.cpp b/src/mongo/db/repl/isself.cpp
index 243a1c065f0..a9f776e5f74 100644
--- a/src/mongo/db/repl/isself.cpp
+++ b/src/mongo/db/repl/isself.cpp
@@ -163,10 +163,17 @@ std::vector<std::string> getAddrsForHost(const std::string& iporhost,
 
 }  // namespace
 
-bool isSelf(const HostAndPort& hostAndPort, ServiceContext* const ctx) {
+bool isSelf(const HostAndPort& hostAndPort, ServiceContext* const ctx, Milliseconds timeout) {
+    if (isSelfFastPath(hostAndPort)) {
+        return true;
+    }
+    return isSelfSlowPath(hostAndPort, ctx, timeout);
+}
+
+bool isSelfFastPath(const HostAndPort& hostAndPort) {
     if (MONGO_unlikely(failIsSelfCheck.shouldFail())) {
         LOGV2(356490,
-              "failIsSelfCheck failpoint activated, returning false from isSelf",
+              "failIsSelfCheck failpoint activated, returning false from isSelfFastPath",
               "hostAndPort"_attr = hostAndPort);
         return false;
     }
@@ -222,12 +229,24 @@ bool isSelf(const HostAndPort& hostAndPort, ServiceContext* const ctx) {
             }
         }
     }
+    return false;
+}
 
+bool isSelfSlowPath(const HostAndPort& hostAndPort,
+                    ServiceContext* const ctx,
+                    Milliseconds timeout) {
     ctx->waitForStartupComplete();
+    if (MONGO_unlikely(failIsSelfCheck.shouldFail())) {
+        LOGV2(6605000,
+              "failIsSelfCheck failpoint activated, returning false from isSelfSlowPath",
+              "hostAndPort"_attr = hostAndPort);
+        return false;
+    }
 
     try {
         DBClientConnection conn;
-        conn.setSoTimeout(30);  // 30 second timeout
+        double timeoutSeconds = static_cast<double>(durationCount<Milliseconds>(timeout)) / 1000.0;
+        conn.setSoTimeout(timeoutSeconds);
 
         // We need to avoid the isMaster call triggered by a normal connect, which would
         // cause a deadlock. 'isSelf' is called by the Replication Coordinator when validating
diff --git a/src/mongo/db/repl/isself.h b/src/mongo/db/repl/isself.h
index 84264978720..fb61c8a6cb3 100644
--- a/src/mongo/db/repl/isself.h
+++ b/src/mongo/db/repl/isself.h
@@ -33,6 +33,7 @@
 #include <vector>
 
 #include "mongo/bson/oid.h"
+#include "mongo/util/duration.h"
 
 namespace mongo {
 struct HostAndPort;
@@ -49,7 +50,21 @@ extern OID instanceId;
 /**
  * Returns true if "hostAndPort" identifies this instance.
  */
-bool isSelf(const HostAndPort& hostAndPort, ServiceContext* ctx);
+bool isSelf(const HostAndPort& hostAndPort,
+            ServiceContext* ctx,
+            Milliseconds timeout = Seconds(30));
+
+/**
+ * Returns true if "hostAndPort" identifies this instance by checking our bound IP addresses,
+ * without going out to the network and running the _isSelf command on the node.
+ */
+bool isSelfFastPath(const HostAndPort& hostAndPort);
+
+/**
+ * Returns true if "hostAndPort" identifies this instance by running the _isSelf command on the
+ * node.
+ */
+bool isSelfSlowPath(const HostAndPort& hostAndPort, ServiceContext* ctx, Milliseconds timeout);
 
 /**
  * Returns all the IP addresses bound to the network interfaces of this machine.
diff --git a/src/mongo/db/repl/isself_test.cpp b/src/mongo/db/repl/isself_test.cpp
index 511ef122ecf..91ad0ad3803 100644
--- a/src/mongo/db/repl/isself_test.cpp
+++ b/src/mongo/db/repl/isself_test.cpp
@@ -55,6 +55,7 @@ TEST_F(ServiceContextTest, DetectsSameHostIPv4) {
     // Fastpath should agree with the result of getBoundAddrs
     // since it uses it...
     for (std::vector<string>::const_iterator it = addrs.begin(); it != addrs.end(); ++it) {
+        ASSERT(isSelfFastPath(HostAndPort(*it, serverGlobalParams.port)));
         ASSERT(isSelf(HostAndPort(*it, serverGlobalParams.port), getGlobalServiceContext()));
     }
 #else
@@ -72,6 +73,7 @@ TEST_F(ServiceContextTest, DetectsSameHostIPv6) {
     // Fastpath should agree with the result of getBoundAddrs
     // since it uses it...
     for (std::vector<string>::const_iterator it = addrs.begin(); it != addrs.end(); ++it) {
+        ASSERT(isSelfFastPath(HostAndPort(*it, serverGlobalParams.port)));
         ASSERT(isSelf(HostAndPort(*it, serverGlobalParams.port), getGlobalServiceContext()));
     }
 #else
diff --git a/src/mongo/db/repl/repl_set_config_checks.cpp b/src/mongo/db/repl/repl_set_config_checks.cpp
index 3be0e88e5d8..cb769f382f8 100644
--- a/src/mongo/db/repl/repl_set_config_checks.cpp
+++ b/src/mongo/db/repl/repl_set_config_checks.cpp
@@ -321,11 +321,21 @@ StatusWith<int> findSelfInConfig(ReplicationCoordinatorExternalState* externalSt
     for (ReplSetConfig::MemberIterator iter = newConfig.membersBegin();
          iter != newConfig.membersEnd();
          ++iter) {
-        if (externalState->isSelf(iter->getHostAndPort(), ctx)) {
+        if (externalState->isSelfFastPath(iter->getHostAndPort())) {
             meConfigs.push_back(iter);
         }
     }
     if (meConfigs.empty()) {
+        // No self-hosts were found using the fastpath; check with the slow path.
+        for (ReplSetConfig::MemberIterator iter = newConfig.membersBegin();
+             iter != newConfig.membersEnd();
+             ++iter) {
+            if (externalState->isSelfSlowPath(iter->getHostAndPort(), ctx, Seconds(30))) {
+                meConfigs.push_back(iter);
+            }
+        }
+    }
+    if (meConfigs.empty()) {
         return StatusWith<int>(ErrorCodes::NodeNotFound,
                                str::stream() << "No host described in new configuration with "
                                              << newConfig.getConfigVersionAndTerm().toString()
diff --git a/src/mongo/db/repl/repl_set_config_checks_test.cpp b/src/mongo/db/repl/repl_set_config_checks_test.cpp
index 21cb645cf10..d491bdd9131 100644
--- a/src/mongo/db/repl/repl_set_config_checks_test.cpp
+++ b/src/mongo/db/repl/repl_set_config_checks_test.cpp
@@ -1288,6 +1288,60 @@ TEST_F(ServiceContextTest, FindSelfInConfig) {
             .getStatus());
 }
 
+TEST_F(ServiceContextTest, FindSelfInConfigFastAndSlow) {
+    ReplSetConfig newConfig;
+    newConfig = ReplSetConfig::parse(BSON("_id"
+                                          << "rs0"
+                                          << "version" << 2 << "protocolVersion" << 1 << "members"
+                                          << BSON_ARRAY(BSON("_id" << 1 << "host"
+                                                                   << "h1")
+                                                        << BSON("_id" << 2 << "host"
+                                                                      << "h2")
+                                                        << BSON("_id" << 3 << "host"
+                                                                      << "h3"))));
+
+    {
+        // Present once only on the slow path, but fast enough to be found
+        ReplicationCoordinatorExternalStateMock presentOnceExternalState;
+        presentOnceExternalState.addSelfSlow(HostAndPort("h2"), Seconds(29));
+        ASSERT_EQUALS(1,
+                      unittest::assertGet(findSelfInConfig(
+                          &presentOnceExternalState, newConfig, getServiceContext())));
+    }
+
+    {
+        // Present twice only on the slow path, but fast enough to be found both times.
+        ReplicationCoordinatorExternalStateMock presentTwiceExternalState;
+        presentTwiceExternalState.addSelfSlow(HostAndPort("h2"), Seconds(29));
+        presentTwiceExternalState.addSelfSlow(HostAndPort("h3"), Seconds(29));
+        ASSERT_EQUALS(ErrorCodes::InvalidReplicaSetConfig,
+                      findSelfInConfig(&presentTwiceExternalState, newConfig, getServiceContext())
+                          .getStatus());
+    }
+
+    {
+        // Present once on the fast path, once on the slow path. This is expected to erroneously
+        // succeed, because we should not check the slow path if we got a unique result on the fast
+        // path.
+        ReplicationCoordinatorExternalStateMock presentFastAndSlowExternalState;
+        presentFastAndSlowExternalState.addSelf(HostAndPort("h2"));
+        presentFastAndSlowExternalState.addSelfSlow(HostAndPort("h3"), Seconds(29));
+        ASSERT_EQUALS(1,
+                      unittest::assertGet(findSelfInConfig(
+                          &presentFastAndSlowExternalState, newConfig, getServiceContext())));
+    }
+
+    {
+        // Present only on the slow path, with a long timeout.  This will fail.
+        ReplicationCoordinatorExternalStateMock presentLongTimeoutExternalState;
+        presentLongTimeoutExternalState.addSelfSlow(HostAndPort("h2"), Seconds(31));
+        ASSERT_EQUALS(
+            ErrorCodes::NodeNotFound,
+            findSelfInConfig(&presentLongTimeoutExternalState, newConfig, getServiceContext())
+                .getStatus());
+    }
+}
+
 }  // namespace
 }  // namespace repl
 }  // namespace mongo
diff --git a/src/mongo/db/repl/replication_coordinator_external_state.h b/src/mongo/db/repl/replication_coordinator_external_state.h
index 9bf96582627..eec7564971c 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state.h
+++ b/src/mongo/db/repl/replication_coordinator_external_state.h
@@ -146,6 +146,20 @@ public:
     virtual bool isSelf(const HostAndPort& host, ServiceContext* service) = 0;
 
     /**
+     * Returns true if "host" is one of the network identities of this node, without actually
+     * going out to the network and checking.
+     */
+    virtual bool isSelfFastPath(const HostAndPort& host) = 0;
+
+    /**
+     * Returns true if "host" is one of the network identities of this node, without
+     * checking the fast path first.
+     */
+    virtual bool isSelfSlowPath(const HostAndPort& host,
+                                ServiceContext* service,
+                                Milliseconds timeout) = 0;
+
+    /**
      * Gets the replica set config document from local storage, or returns an error.
      */
     virtual StatusWith<BSONObj> loadLocalConfigDocument(OperationContext* opCtx) = 0;
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
index 85bdd8d307c..f8982e50af2 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
@@ -805,6 +805,16 @@ bool ReplicationCoordinatorExternalStateImpl::isSelf(const HostAndPort& host, Se
     return repl::isSelf(host, ctx);
 }
 
+bool ReplicationCoordinatorExternalStateImpl::isSelfFastPath(const HostAndPort& host) {
+    return repl::isSelfFastPath(host);
+}
+
+bool ReplicationCoordinatorExternalStateImpl::isSelfSlowPath(const HostAndPort& host,
+                                                             ServiceContext* ctx,
+                                                             Milliseconds timeout) {
+    return repl::isSelfSlowPath(host, ctx, timeout);
+}
+
 HostAndPort ReplicationCoordinatorExternalStateImpl::getClientHostAndPort(
     const OperationContext* opCtx) {
     return HostAndPort(opCtx->getClient()->clientAddress(true));
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.h b/src/mongo/db/repl/replication_coordinator_external_state_impl.h
index 9a1e448f636..20eb8c61984 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.h
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.h
@@ -81,6 +81,10 @@ public:
     OpTime onTransitionToPrimary(OperationContext* opCtx) override;
     virtual void forwardSecondaryProgress();
     virtual bool isSelf(const HostAndPort& host, ServiceContext* service);
+    bool isSelfFastPath(const HostAndPort& host) final;
+    bool isSelfSlowPath(const HostAndPort& host,
+                        ServiceContext* service,
+                        Milliseconds timeout) final;
     Status createLocalLastVoteCollection(OperationContext* opCtx) final;
     virtual StatusWith<BSONObj> loadLocalConfigDocument(OperationContext* opCtx);
     virtual Status storeLocalConfigDocument(OperationContext* opCtx,
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp
index 86aaa76c00f..5f449c1ec51 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp
@@ -96,9 +96,24 @@ void ReplicationCoordinatorExternalStateMock::forwardSecondaryProgress() {}
 
 bool ReplicationCoordinatorExternalStateMock::isSelf(const HostAndPort& host,
                                                      ServiceContext* const service) {
+    return sequenceContains(_selfHosts, host) || _selfHostsSlow.find(host) != _selfHostsSlow.end();
+}
+
+bool ReplicationCoordinatorExternalStateMock::isSelfFastPath(const HostAndPort& host) {
     return sequenceContains(_selfHosts, host);
 }
 
+bool ReplicationCoordinatorExternalStateMock::isSelfSlowPath(const HostAndPort& host,
+                                                             ServiceContext* const service,
+                                                             Milliseconds timeout) {
+    if (sequenceContains(_selfHosts, host))
+        return true;
+    auto iter = _selfHostsSlow.find(host);
+    if (iter == _selfHostsSlow.end())
+        return false;
+    return iter->second <= timeout;
+}
+
 void ReplicationCoordinatorExternalStateMock::addSelf(const HostAndPort& host) {
     _selfHosts.push_back(host);
 }
@@ -107,6 +122,11 @@ void ReplicationCoordinatorExternalStateMock::clearSelfHosts() {
     _selfHosts.clear();
 }
 
+void ReplicationCoordinatorExternalStateMock::addSelfSlow(const HostAndPort& host,
+                                                          Milliseconds timeout) {
+    _selfHostsSlow.emplace(host, timeout);
+}
+
 HostAndPort ReplicationCoordinatorExternalStateMock::getClientHostAndPort(
     const OperationContext* opCtx) {
     return _clientHostAndPort;
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.h b/src/mongo/db/repl/replication_coordinator_external_state_mock.h
index ecd0f072fed..fd8327f4fee 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_mock.h
+++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.h
@@ -73,6 +73,10 @@ public:
     OpTime onTransitionToPrimary(OperationContext* opCtx) override;
     virtual void forwardSecondaryProgress();
     virtual bool isSelf(const HostAndPort& host, ServiceContext* service);
+    bool isSelfFastPath(const HostAndPort& host) final;
+    bool isSelfSlowPath(const HostAndPort& host,
+                        ServiceContext* service,
+                        Milliseconds timeout) final;
     virtual HostAndPort getClientHostAndPort(const OperationContext* opCtx);
     virtual StatusWith<BSONObj> loadLocalConfigDocument(OperationContext* opCtx);
     virtual Status storeLocalConfigDocument(OperationContext* opCtx,
@@ -105,13 +109,20 @@ public:
 
     /**
      * Adds "host" to the list of hosts that this mock will match when responding to "isSelf"
-     * messages.
+     * messages, including "isSelfFastPath" and "isSelfSlowPath".
      */
     void addSelf(const HostAndPort& host);
 
     /**
+     * Adds "host" to the list of hosts that this mock will match when responding to
+     * "isSelfSlowPath" messages with a timeout less than or equal to that given,
+     * but not "isSelfFastPath" messages.
+     */
+    void addSelfSlow(const HostAndPort& host, Milliseconds timeout);
+
+    /**
      * Remove all hosts from the list of hosts that this mock will match when responding to "isSelf"
-     * messages.
+     * messages.  Clears both regular and slow hosts.
      */
     void clearSelfHosts();
 
@@ -208,6 +219,7 @@ private:
     StatusWith<OpTime> _lastOpTime;
     StatusWith<Date_t> _lastWallTime;
     std::vector<HostAndPort> _selfHosts;
+    stdx::unordered_map<HostAndPort, Milliseconds> _selfHostsSlow;
     bool _canAcquireGlobalSharedLock;
     Status _storeLocalConfigDocumentStatus;
     Status _storeLocalLastVoteDocumentStatus;