SERVER-44522 serverStatus metrics for awaitable isMaster

2 files changed, 32 insertions, 1 deletions

diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index 5685e7955f2..751f971a648 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -93,6 +93,7 @@
 #include "mongo/platform/mutex.h"
 #include "mongo/rpc/metadata/oplog_query_metadata.h"
 #include "mongo/rpc/metadata/repl_set_metadata.h"
+#include "mongo/transport/ismaster_metrics.h"
 #include "mongo/util/assert_util.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/scopeguard.h"
@@ -109,7 +110,10 @@ MONGO_FAIL_POINT_DEFINE(stepdownHangBeforeRSTLEnqueue);
 // Fail setMaintenanceMode with ErrorCodes::NotSecondary to simulate a concurrent election.
 MONGO_FAIL_POINT_DEFINE(setMaintenanceModeFailsWithNotSecondary);
 MONGO_FAIL_POINT_DEFINE(forceSyncSourceRetryWaitForInitialSync);
+// Signals that an isMaster request has started waiting.
 MONGO_FAIL_POINT_DEFINE(waitForIsMasterResponse);
+// Will cause an isMaster request to hang as it starts waiting.
+MONGO_FAIL_POINT_DEFINE(hangWhileWaitingForIsMasterResponse);
 
 // Number of times we tried to go live as a secondary.
 Counter64 attemptsToBecomeSecondary;
@@ -2062,10 +2066,16 @@ std::shared_ptr<const IsMasterResponse> ReplicationCoordinatorImpl::awaitIsMaste
     const TopologyVersion topologyVersion = _topCoord->getTopologyVersion();
     lk.unlock();
 
+    IsMasterMetrics::get(opCtx)->incrementNumAwaitingTopologyChanges();
+
     if (MONGO_unlikely(waitForIsMasterResponse.shouldFail())) {
         // Used in tests that wait for this failpoint to be entered before triggering a topology
         // change.
-        LOGV2(21341, "waitForIsMasterResponse failpoint enabled.");
+        LOGV2(31464, "waitForIsMasterResponse failpoint enabled.");
+    }
+    if (MONGO_unlikely(hangWhileWaitingForIsMasterResponse.shouldFail())) {
+        LOGV2(21341, "Hanging due to hangWhileWaitingForIsMasterResponse failpoint.");
+        hangWhileWaitingForIsMasterResponse.pauseWhileSet(opCtx);
     }
 
     // Wait for a topology change with timeout set to deadline.
@@ -2083,6 +2093,7 @@ std::shared_ptr<const IsMasterResponse> ReplicationCoordinatorImpl::awaitIsMaste
         // Return an IsMasterResponse with the current topology version on timeout when waiting for
         // a topology change.
         stdx::lock_guard lk(_mutex);
+        IsMasterMetrics::get(opCtx)->decrementNumAwaitingTopologyChanges();
         return _makeIsMasterResponse(horizonString, lk);
     }
 
@@ -3241,6 +3252,8 @@ ReplicationCoordinatorImpl::_updateMemberStateFromTopologyCoordinator(WithLock l
     ON_BLOCK_EXIT([&] {
         if (_rsConfig.isInitialized()) {
             _fulfillTopologyChangePromise(opCtx, lk);
+            // Use the global ServiceContext here in case the current opCtx is null.
+            IsMasterMetrics::get(getGlobalServiceContext())->resetNumAwaitingTopologyChanges();
         }
     });
 
@@ -3736,6 +3749,7 @@ ReplicationCoordinatorImpl::_setCurrentRSConfig(WithLock lk,
         for (auto iter = _horizonToPromiseMap.begin(); iter != _horizonToPromiseMap.end(); iter++) {
             iter->second->setError({ErrorCodes::SplitHorizonChange,
                                     "Received a reconfig that changed the horizon parameters."});
+            IsMasterMetrics::get(opCtx)->resetNumAwaitingTopologyChanges();
         }
         if (_selfIndex >= 0) {
             // Only create a new horizon promise mapping if the node exists in the new config.
diff --git a/src/mongo/db/repl/replication_info.cpp b/src/mongo/db/repl/replication_info.cpp
index d5c183c365f..fd8ce22e8d0 100644
--- a/src/mongo/db/repl/replication_info.cpp
+++ b/src/mongo/db/repl/replication_info.cpp
@@ -63,13 +63,18 @@
 #include "mongo/logv2/log.h"
 #include "mongo/rpc/metadata/client_metadata.h"
 #include "mongo/rpc/metadata/client_metadata_ismaster.h"
+#include "mongo/transport/ismaster_metrics.h"
 #include "mongo/util/decimal_counter.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/map_util.h"
 
 namespace mongo {
 
+// Hangs in the beginning of each isMaster command when set.
 MONGO_FAIL_POINT_DEFINE(waitInIsMaster);
+// Awaitable isMaster requests with the proper topologyVersions will sleep for maxAwaitTimeMS on
+// standalones. This failpoint will hang right before doing this sleep when set.
+MONGO_FAIL_POINT_DEFINE(hangWaitingForIsMasterResponseOnStandalone);
 
 using std::list;
 using std::string;
@@ -121,6 +126,15 @@ TopologyVersion appendReplicationInfo(OperationContext* opCtx,
         // The topologyVersion never changes on a running standalone process, so just sleep for
         // maxAwaitTimeMS.
         invariant(maxAwaitTimeMS);
+
+        IsMasterMetrics::get(opCtx)->incrementNumAwaitingTopologyChanges();
+        ON_BLOCK_EXIT([&] { IsMasterMetrics::get(opCtx)->decrementNumAwaitingTopologyChanges(); });
+        if (MONGO_unlikely(hangWaitingForIsMasterResponseOnStandalone.shouldFail())) {
+            // Used in tests that wait for this failpoint to be entered to guarantee that the
+            // request is waiting and metrics have been updated.
+            LOGV2(31462, "Hanging due to hangWaitingForIsMasterResponseOnStandalone failpoint.");
+            hangWaitingForIsMasterResponseOnStandalone.pauseWhileSet(opCtx);
+        }
         opCtx->sleepFor(Milliseconds(*maxAwaitTimeMS));
     }
 
@@ -499,6 +513,9 @@ public:
                     maxAwaitTimeMSField);
             invariant(clientTopologyVersion);
 
+            InExhaustIsMaster::get(opCtx->getClient()->session().get())
+                ->setInExhaustIsMaster(true /* inExhaustIsMaster */);
+
             if (clientTopologyVersion->getProcessId() == currentTopologyVersion.getProcessId() &&
                 clientTopologyVersion->getCounter() == currentTopologyVersion.getCounter()) {
                 // Indicate that an exhaust message should be generated and the previous BSONObj