SERVER-28664 Use pool connections in MRU order

1 files changed, 174 insertions, 0 deletions

diff --git a/src/mongo/executor/connection_pool_test.cpp b/src/mongo/executor/connection_pool_test.cpp
index 6a857602ca6..a3b5cca24a8 100644
--- a/src/mongo/executor/connection_pool_test.cpp
+++ b/src/mongo/executor/connection_pool_test.cpp
@@ -27,12 +27,17 @@
 
 #include "mongo/platform/basic.h"
 
+#include <algorithm>
+#include <random>
+#include <stack>
+
 #include "mongo/executor/connection_pool_test_fixture.h"
 
 #include "mongo/executor/connection_pool.h"
 #include "mongo/stdx/future.h"
 #include "mongo/stdx/memory.h"
 #include "mongo/unittest/unittest.h"
+#include "mongo/util/scopeguard.h"
 
 namespace mongo {
 namespace executor {
@@ -95,6 +100,175 @@ TEST_F(ConnectionPoolTest, SameConn) {
 }
 
 /**
+ * Verify that connections are obtained in MRU order.
+ */
+TEST_F(ConnectionPoolTest, ConnectionsAreAcquiredInMRUOrder) {
+    ConnectionPool pool(stdx::make_unique<PoolImpl>(), "test pool");
+
+    // Obtain a set of connections
+    constexpr size_t kSize = 100;
+    std::vector<ConnectionPool::ConnectionHandle> connections;
+
+    // Ensure that no matter how we leave the test, we mark any
+    // checked out connections as OK before implicity returning them
+    // to the pool by destroying the 'connections' vector. Otherwise,
+    // this test would cause an invariant failure instead of a normal
+    // test failure if it fails, which would be confusing.
+    const auto guard = MakeGuard([&] {
+        while (!connections.empty()) {
+            try {
+                ConnectionPool::ConnectionHandle conn = std::move(connections.back());
+                connections.pop_back();
+                conn->indicateSuccess();
+            } catch (...) {
+            }
+        }
+    });
+
+    for (size_t i = 0; i != kSize; ++i) {
+        ConnectionImpl::pushSetup(Status::OK());
+        pool.get(HostAndPort(),
+                 Milliseconds(5000),
+                 [&](StatusWith<ConnectionPool::ConnectionHandle> swConn) {
+                     ASSERT(swConn.isOK());
+                     connections.push_back(std::move(swConn.getValue()));
+                 });
+    }
+
+    // Shuffle them into a random order
+    std::random_device rd;
+    std::mt19937 rng(rd());
+    std::shuffle(connections.begin(), connections.end(), rng);
+
+    // Return them to the pool in that random order, recording IDs in a stack
+    std::stack<size_t> ids;
+    while (!connections.empty()) {
+        ConnectionPool::ConnectionHandle conn = std::move(connections.back());
+        connections.pop_back();
+        ids.push(static_cast<ConnectionImpl*>(conn.get())->id());
+        conn->indicateSuccess();
+    }
+
+    // Re-obtain the connections. They should come back in the same order
+    // as the IDs in the stack, since the pool returns them in MRU order.
+    for (size_t i = 0; i != kSize; ++i) {
+        ConnectionImpl::pushSetup(Status::OK());
+        pool.get(HostAndPort(),
+                 Milliseconds(5000),
+                 [&](StatusWith<ConnectionPool::ConnectionHandle> swConn) {
+                     ASSERT(swConn.isOK());
+                     const auto id = CONN2ID(swConn);
+                     connections.push_back(std::move(swConn.getValue()));
+                     ASSERT(id == ids.top());
+                     ids.pop();
+                 });
+    }
+}
+
+/**
+ * Verify that recently used connections are not purged.
+ */
+TEST_F(ConnectionPoolTest, ConnectionsNotUsedRecentlyArePurged) {
+    ConnectionPool::Options options;
+    options.minConnections = 0;
+    options.refreshRequirement = Milliseconds(1000);
+    options.refreshTimeout = Milliseconds(5000);
+    options.hostTimeout = Minutes(1);
+    ConnectionPool pool(stdx::make_unique<PoolImpl>(), "test pool", options);
+
+    ASSERT_EQ(pool.getNumConnectionsPerHost(HostAndPort()), 0U);
+
+    // Obtain a set of connections
+    constexpr size_t kSize = 100;
+    std::vector<ConnectionPool::ConnectionHandle> connections;
+
+    // Ensure that no matter how we leave the test, we mark any
+    // checked out connections as OK before implicity returning them
+    // to the pool by destroying the 'connections' vector. Otherwise,
+    // this test would cause an invariant failure instead of a normal
+    // test failure if it fails, which would be confusing.
+    const auto guard = MakeGuard([&] {
+        while (!connections.empty()) {
+            try {
+                ConnectionPool::ConnectionHandle conn = std::move(connections.back());
+                connections.pop_back();
+                conn->indicateSuccess();
+            } catch (...) {
+            }
+        }
+    });
+
+    auto now = Date_t::now();
+    PoolImpl::setNow(now);
+
+    // Check out kSize connections from the pool, and record their IDs in a set.
+    std::set<size_t> original_ids;
+    for (size_t i = 0; i != kSize; ++i) {
+        ConnectionImpl::pushSetup(Status::OK());
+        pool.get(HostAndPort(),
+                 Milliseconds(5000),
+                 [&](StatusWith<ConnectionPool::ConnectionHandle> swConn) {
+                     ASSERT(swConn.isOK());
+                     original_ids.insert(CONN2ID(swConn));
+                     connections.push_back(std::move(swConn.getValue()));
+                 });
+    }
+
+    ASSERT_EQ(original_ids.size(), kSize);
+    ASSERT_EQ(pool.getNumConnectionsPerHost(HostAndPort()), kSize);
+
+    // Shuffle them into a random order
+    std::random_device rd;
+    std::mt19937 rng(rd());
+    std::shuffle(connections.begin(), connections.end(), rng);
+
+    // Return them to the pool in that random order.
+    while (!connections.empty()) {
+        ConnectionPool::ConnectionHandle conn = std::move(connections.back());
+        connections.pop_back();
+        conn->indicateSuccess();
+    }
+
+    // Advance the time, but not enough to age out connections. We should still have them all.
+    PoolImpl::setNow(now + Milliseconds(500));
+    ASSERT_EQ(pool.getNumConnectionsPerHost(HostAndPort()), kSize);
+
+    // Re-obtain a quarter of the connections, and record their IDs in a set.
+    std::set<size_t> reacquired_ids;
+    for (size_t i = 0; i < kSize / 4; ++i) {
+        ConnectionImpl::pushSetup(Status::OK());
+        pool.get(HostAndPort(),
+                 Milliseconds(5000),
+                 [&](StatusWith<ConnectionPool::ConnectionHandle> swConn) {
+                     ASSERT(swConn.isOK());
+                     reacquired_ids.insert(CONN2ID(swConn));
+                     connections.push_back(std::move(swConn.getValue()));
+                 });
+    }
+
+    ASSERT_EQ(reacquired_ids.size(), kSize / 4);
+    ASSERT(std::includes(
+        original_ids.begin(), original_ids.end(), reacquired_ids.begin(), reacquired_ids.end()));
+    ASSERT_EQ(pool.getNumConnectionsPerHost(HostAndPort()), kSize);
+
+    // Put them right back in.
+    while (!connections.empty()) {
+        ConnectionPool::ConnectionHandle conn = std::move(connections.back());
+        connections.pop_back();
+        conn->indicateSuccess();
+    }
+
+    // We should still have all of them in the pool
+    ASSERT_EQ(pool.getNumConnectionsPerHost(HostAndPort()), kSize);
+
+    // Advance across the host timeout for the 75 connections we
+    // didn't use. Afterwards, the pool should contain only those
+    // kSize/4 connections we used above.
+    PoolImpl::setNow(now + Milliseconds(1000));
+    ASSERT_EQ(pool.getNumConnectionsPerHost(HostAndPort()), kSize / 4);
+}
+
+/**
  * Verify that a failed connection isn't returned to the pool
  */
 TEST_F(ConnectionPoolTest, FailedConnDifferentConn) {