SERVER-55055 Add cumulative metric for the total refreshed connections

3 files changed, 48 insertions, 10 deletions

diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp
index 1fa7790a300..9e562057d38 100644
--- a/src/mongo/executor/connection_pool.cpp
+++ b/src/mongo/executor/connection_pool.cpp
@@ -61,6 +61,8 @@ namespace mongo {
 
 namespace {
 
+MONGO_FAIL_POINT_DEFINE(refreshConnectionAfterEveryCommand);
+
 auto makeSeveritySuppressor() {
     return std::make_unique<logv2::KeyedSeveritySuppressor<HostAndPort>>(
         Seconds{1}, logv2::LogSeverity::Log(), logv2::LogSeverity::Debug(2));
@@ -306,6 +308,11 @@ public:
     size_t refreshingConnections() const;
 
     /**
+     * Returns the number of all refreshed connections in the pool.
+     */
+    size_t refreshedConnections() const;
+
+    /**
      * Returns the total number of connections ever created in this pool.
      */
     size_t createdConnections() const;
@@ -428,6 +435,8 @@ private:
 
     size_t _created = 0;
 
+    size_t _refreshed = 0;
+
     transport::Session::TagMask _tags = transport::Session::kPending;
 
     HostHealth _health;
@@ -582,7 +591,8 @@ void ConnectionPool::appendConnectionStats(ConnectionPoolStats* stats) const {
         ConnectionStatsPer hostStats{pool->inUseConnections(),
                                      pool->availableConnections(),
                                      pool->createdConnections(),
-                                     pool->refreshingConnections()};
+                                     pool->refreshingConnections(),
+                                     pool->refreshedConnections()};
         stats->updateStatsForHost(_name, host, hostStats);
     }
 }
@@ -630,6 +640,10 @@ size_t ConnectionPool::SpecificPool::refreshingConnections() const {
     return _processingPool.size();
 }
 
+size_t ConnectionPool::SpecificPool::refreshedConnections() const {
+    return _refreshed;
+}
+
 size_t ConnectionPool::SpecificPool::createdConnections() const {
     return _created;
 }
@@ -730,6 +744,10 @@ ConnectionPool::ConnectionHandle ConnectionPool::SpecificPool::tryGetConnection(
 void ConnectionPool::SpecificPool::finishRefresh(ConnectionInterface* connPtr, Status status) {
     auto conn = takeFromProcessingPool(connPtr);
 
+    // We increment the total number of refreshed connections right upfront to track all completed
+    // refreshes.
+    _refreshed++;
+
     // If we're in shutdown, we don't need refreshed connections
     if (_health.isShutdown) {
         return;
@@ -814,10 +832,15 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr
         return;
     }
 
-    auto now = _parent->_factory->now();
-    if (needsRefreshTP <= now) {
-        // If we need to refresh this connection
+    // If we need to refresh this connection
+    bool shouldRefreshConnection = needsRefreshTP <= _parent->_factory->now();
+
+    if (MONGO_unlikely(refreshConnectionAfterEveryCommand.shouldFail())) {
+        LOGV2(5505501, "refresh connection after every command is on");
+        shouldRefreshConnection = true;
+    }
 
+    if (shouldRefreshConnection) {
         auto controls = _parent->_controller->getControls(_id);
         if (_readyPool.size() + _processingPool.size() + _checkedOutPool.size() >=
             controls.targetConnections) {
diff --git a/src/mongo/executor/connection_pool_stats.cpp b/src/mongo/executor/connection_pool_stats.cpp
index 843a19a1a27..a3d1e1fb0b1 100644
--- a/src/mongo/executor/connection_pool_stats.cpp
+++ b/src/mongo/executor/connection_pool_stats.cpp
@@ -36,11 +36,13 @@
 namespace mongo {
 namespace executor {
 
-ConnectionStatsPer::ConnectionStatsPer(size_t nInUse,
-                                       size_t nAvailable,
-                                       size_t nCreated,
-                                       size_t nRefreshing)
-    : inUse(nInUse), available(nAvailable), created(nCreated), refreshing(nRefreshing) {}
+ConnectionStatsPer::ConnectionStatsPer(
+    size_t nInUse, size_t nAvailable, size_t nCreated, size_t nRefreshing, size_t nRefreshed)
+    : inUse(nInUse),
+      available(nAvailable),
+      created(nCreated),
+      refreshing(nRefreshing),
+      refreshed(nRefreshed) {}
 
 ConnectionStatsPer::ConnectionStatsPer() = default;
 
@@ -49,6 +51,7 @@ ConnectionStatsPer& ConnectionStatsPer::operator+=(const ConnectionStatsPer& oth
     available += other.available;
     created += other.created;
     refreshing += other.refreshing;
+    refreshed += other.refreshed;
 
     return *this;
 }
@@ -74,6 +77,7 @@ void ConnectionPoolStats::updateStatsForHost(std::string pool,
     totalAvailable += newStats.available;
     totalCreated += newStats.created;
     totalRefreshing += newStats.refreshing;
+    totalRefreshed += newStats.refreshed;
 }
 
 void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result, bool forFTDC) {
@@ -81,6 +85,7 @@ void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result, bool forFT
     result.appendNumber("totalAvailable", static_cast<long long>(totalAvailable));
     result.appendNumber("totalCreated", static_cast<long long>(totalCreated));
     result.appendNumber("totalRefreshing", static_cast<long long>(totalRefreshing));
+    result.appendNumber("totalRefreshed", static_cast<long long>(totalRefreshed));
 
     if (forFTDC) {
         BSONObjBuilder poolBuilder(result.subobjStart("connectionsInUsePerPool"));
@@ -98,6 +103,7 @@ void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result, bool forFT
         return;
     }
 
+    // Process pools stats.
     {
         if (strategy) {
             result.append("replicaSetMatchingStrategy", matchingStrategyToString(*strategy));
@@ -111,6 +117,7 @@ void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result, bool forFT
             poolInfo.appendNumber("poolAvailable", static_cast<long long>(poolStats.available));
             poolInfo.appendNumber("poolCreated", static_cast<long long>(poolStats.created));
             poolInfo.appendNumber("poolRefreshing", static_cast<long long>(poolStats.refreshing));
+            poolInfo.appendNumber("poolRefreshed", static_cast<long long>(poolStats.refreshed));
 
             for (const auto& host : poolStats.statsByHost) {
                 BSONObjBuilder hostInfo(poolInfo.subobjStart(host.first.toString()));
@@ -119,9 +126,12 @@ void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result, bool forFT
                 hostInfo.appendNumber("available", static_cast<long long>(hostStats.available));
                 hostInfo.appendNumber("created", static_cast<long long>(hostStats.created));
                 hostInfo.appendNumber("refreshing", static_cast<long long>(hostStats.refreshing));
+                hostInfo.appendNumber("refreshed", static_cast<long long>(hostStats.refreshed));
             }
         }
     }
+
+    // Processes hosts stats.
     {
         BSONObjBuilder hostBuilder(result.subobjStart("hosts"));
         for (auto&& host : statsByHost) {
@@ -131,6 +141,7 @@ void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result, bool forFT
             hostInfo.appendNumber("available", static_cast<long long>(hostStats.available));
             hostInfo.appendNumber("created", static_cast<long long>(hostStats.created));
             hostInfo.appendNumber("refreshing", static_cast<long long>(hostStats.refreshing));
+            hostInfo.appendNumber("refreshed", static_cast<long long>(hostStats.refreshed));
         }
     }
 }
diff --git a/src/mongo/executor/connection_pool_stats.h b/src/mongo/executor/connection_pool_stats.h
index 3f9235040eb..a96739b42be 100644
--- a/src/mongo/executor/connection_pool_stats.h
+++ b/src/mongo/executor/connection_pool_stats.h
@@ -41,7 +41,8 @@ namespace executor {
  * a parent ConnectionPoolStats object and should not need to be created directly.
  */
 struct ConnectionStatsPer {
-    ConnectionStatsPer(size_t nInUse, size_t nAvailable, size_t nCreated, size_t nRefreshing);
+    ConnectionStatsPer(
+        size_t nInUse, size_t nAvailable, size_t nCreated, size_t nRefreshing, size_t nRefreshed);
 
     ConnectionStatsPer();
 
@@ -51,6 +52,7 @@ struct ConnectionStatsPer {
     size_t available = 0u;
     size_t created = 0u;
     size_t refreshing = 0u;
+    size_t refreshed = 0u;
 };
 
 /**
@@ -61,12 +63,14 @@ struct ConnectionStatsPer {
 struct ConnectionPoolStats {
     void updateStatsForHost(std::string pool, HostAndPort host, ConnectionStatsPer newStats);
 
+    // FTDC : Full Time Diagnostic Data Collection
     void appendToBSON(mongo::BSONObjBuilder& result, bool forFTDC = false);
 
     size_t totalInUse = 0u;
     size_t totalAvailable = 0u;
     size_t totalCreated = 0u;
     size_t totalRefreshing = 0u;
+    size_t totalRefreshed = 0u;
     boost::optional<ShardingTaskExecutorPoolController::MatchingStrategy> strategy;
 
     using StatsByHost = std::map<HostAndPort, ConnectionStatsPer>;