SERVER-37241 Add testing to verify proper expiration of sessions in the sessions collection

(cherry picked from commit 3caa3c4a4be7b84823f22f481365f58b124d6d00)

6 files changed, 496 insertions, 8 deletions

diff --git a/jstests/libs/pin_getmore_cursor.js b/jstests/libs/pin_getmore_cursor.js
new file mode 100644
index 00000000000..ed1cbc02be4
--- /dev/null
+++ b/jstests/libs/pin_getmore_cursor.js
@@ -0,0 +1,63 @@
+/**
+ * Pins a cursor in a seperate shell and then runs the given function.
+ * 'conn': a connection to an instance of a mongod or mongos.
+ * 'sessionId': The id present if the database is currently in a session.
+ * 'dbName': the database to use with the cursor.
+ * 'assertFunction': a function containing the test to be run after a cursor is pinned and hanging.
+ * 'runGetMoreFunc': A function to generate a string that will be executed in the parallel shell.
+ * 'failPointName': The string name of the failpoint where the cursor will hang. The function turns
+ * the failpoint on, the assert function should turn it off whenever it is appropriate for the test.
+ * 'failPointReachedMessage': The message that should be found in the logs when the failpoint has
+ * been reached.
+ */
+
+load("jstests/libs/check_log.js");
+
+function withPinnedCursor(
+    {conn, sessionId, db, assertFunction, runGetMoreFunc, failPointName, failPointReachedMessage}) {
+    // This test runs manual getMores using different connections, which will not inherit the
+    // implicit session of the cursor establishing command.
+    TestData.disableImplicitSessions = true;
+
+    const coll = db.jstest_with_pinned_cursor;
+    coll.drop();
+    db.active_cursor_sentinel.drop();
+    for (let i = 0; i < 100; ++i) {
+        assert.writeOK(coll.insert({value: i}));
+    }
+    let cleanup = null;
+    try {
+        // Enable the specified failpoint.
+        assert.commandWorked(
+            db.adminCommand({configureFailPoint: failPointName, mode: "alwaysOn"}));
+
+        // Issue an initial find in order to create a cursor and obtain its cursorID.
+        let cmdRes = db.runCommand({find: coll.getName(), batchSize: 2});
+        assert.commandWorked(cmdRes);
+        const cursorId = cmdRes.cursor.id;
+        assert.neq(cursorId, NumberLong(0));
+
+        // Let the cursor hang in a different shell with the information it needs to do a getMore.
+        let code = "let cursorId = " + cursorId.toString() + ";";
+        code += "let collName = '" + coll.getName() + "';";
+        if (sessionId) {
+            code += "let sessionId = " + tojson(sessionId) + ";";
+        }
+        code += "(" + runGetMoreFunc.toString() + ")();";
+        code += "db.active_cursor_sentinel.insert({});";
+
+        cleanup = startParallelShell(code, conn.port);
+
+        // Wait until we know the failpoint has been reached.
+        checkLog.contains(conn, failPointReachedMessage);
+        assertFunction(cursorId, coll);
+
+        // Eventually the cursor should be cleaned up.
+        assert.commandWorked(db.adminCommand({configureFailPoint: failPointName, mode: "off"}));
+
+    } finally {
+        if (cleanup) {
+            cleanup();
+        }
+    }
+}
diff --git a/jstests/noPassthrough/verify_sessions_expiration.js b/jstests/noPassthrough/verify_sessions_expiration.js
new file mode 100644
index 00000000000..f6d5166ca80
--- /dev/null
+++ b/jstests/noPassthrough/verify_sessions_expiration.js
@@ -0,0 +1,135 @@
+// Tests valid coordination of the expiration and vivification of documents between the
+// config.system.sessions collection and the logical session cache.
+//
+// 1. Sessions should be removed from the logical session cache when they expire from
+//    the config.system.sessions collection.
+// 2. getMores run on open cursors should update the lastUse field on documents in the
+//    config.system.sessions collection, prolonging the time for expiration on said document
+//    and corresponding session.
+// 3. Open cursors that are not currently in use should be killed when their corresponding sessions
+//    expire from the config.system.sessions collection.
+// 4. Currently running operations corresponding to a session should prevent said session from
+//    expiring from the config.system.sessions collection. If the expiration date has been reached
+//    during a currently running operation, the logical session cache should vivify the session and
+//    replace it in the config.system.sessions collection.
+
+(function() {
+    "use strict";
+
+    // This test makes assertions about the number of logical session records.
+    TestData.disableImplicitSessions = true;
+
+    load("jstests/libs/pin_getmore_cursor.js");  // For "withPinnedCursor".
+
+    const refresh = {refreshLogicalSessionCacheNow: 1};
+    const startSession = {startSession: 1};
+    const failPointName = "keepCursorPinnedDuringGetMore";
+    const failpointReachedMessage = "getMore - keepCursorPinnedDuringGetMore fail point enabled";
+
+    function refreshSessionsAndVerifyCount(config, expectedCount) {
+        config.runCommand(refresh);
+        assert.eq(config.system.sessions.count(), expectedCount);
+    }
+
+    function getSessions(config) {
+        return config.system.sessions.aggregate([{'$listSessions': {allUsers: true}}]).toArray();
+    }
+
+    function verifyOpenCursorCount(db, expectedCount) {
+        assert.eq(db.serverStatus().metrics.cursor.open.total, expectedCount);
+    }
+
+    const dbName = "test";
+    const testCollName = "verify_sessions_find_get_more";
+
+    let conn = MongoRunner.runMongod();
+    let db = conn.getDB(dbName);
+    let config = conn.getDB("config");
+
+    // 1. Verify that sessions expire from config.system.sessions after the timeout has passed.
+    for (let i = 0; i < 5; i++) {
+        let res = db.runCommand(startSession);
+        assert.commandWorked(res, "unable to start session");
+    }
+    refreshSessionsAndVerifyCount(config, 5);
+
+    // Manually delete entries in config.system.sessions to simulate TTL expiration.
+    assert.writeOK(config.system.sessions.remove({}));
+    refreshSessionsAndVerifyCount(config, 0);
+
+    // 2. Verify that getMores after finds will update the 'lastUse' field on documents in the
+    // config.system.sessions collection.
+    for (let i = 0; i < 10; i++) {
+        db[testCollName].insert({_id: i, a: i, b: 1});
+    }
+
+    let cursors = [];
+    for (let i = 0; i < 5; i++) {
+        let session = db.getMongo().startSession({});
+        assert.commandWorked(session.getDatabase("admin").runCommand({usersInfo: 1}),
+                             "initialize the session");
+        cursors.push(session.getDatabase(dbName)[testCollName].find({b: 1}).batchSize(1));
+        assert(cursors[i].hasNext());
+    }
+
+    refreshSessionsAndVerifyCount(config, 5);
+    verifyOpenCursorCount(config, 5);
+
+    let sessionsCollectionArray;
+    let lastUseValues = [];
+    for (let i = 0; i < 3; i++) {
+        for (let j = 0; j < cursors.length; j++) {
+            cursors[j].next();
+        }
+
+        refreshSessionsAndVerifyCount(config, 5);
+        verifyOpenCursorCount(config, 5);
+
+        sessionsCollectionArray = getSessions(config);
+
+        if (i == 0) {
+            for (let j = 0; j < sessionsCollectionArray.length; j++) {
+                lastUseValues.push(sessionsCollectionArray[j].lastUse);
+            }
+        } else {
+            for (let j = 0; j < sessionsCollectionArray.length; j++) {
+                assert.gt(sessionsCollectionArray[j].lastUse, lastUseValues[j]);
+                lastUseValues[j] = sessionsCollectionArray[j].lastUse;
+            }
+        }
+    }
+
+    // 3. Verify that letting sessions expire (simulated by manual deletion) will kill their
+    // cursors.
+    assert.writeOK(config.system.sessions.remove({}));
+
+    refreshSessionsAndVerifyCount(config, 0);
+    verifyOpenCursorCount(config, 0);
+
+    for (let i = 0; i < cursors.length; i++) {
+        assert.commandFailedWithCode(
+            db.runCommand({getMore: cursors[i]._cursor._cursorid, collection: testCollName}),
+            ErrorCodes.CursorNotFound,
+            'expected getMore to fail because the cursor was killed');
+    }
+
+    // 4. Verify that an expired session (simulated by manual deletion) that has a currently running
+    // operation will be vivified during the logical session cache refresh.
+    let pinnedCursorSession = db.getMongo().startSession();
+    withPinnedCursor({
+        conn: conn,
+        db: pinnedCursorSession.getDatabase(dbName),
+        assertFunction: () => {
+            config.system.sessions.remove({});
+            refreshSessionsAndVerifyCount(config, 1);
+            verifyOpenCursorCount(config, 1);
+        },
+        runGetMoreFunc: () => {
+            assert.commandWorked(db.runCommand({getMore: cursorId, collection: collName}));
+        },
+        failPointName: failPointName,
+        failPointReachedMessage: failpointReachedMessage
+    });
+
+    MongoRunner.stopMongod(conn);
+})();
diff --git a/jstests/replsets/verify_sessions_expiration_rs.js b/jstests/replsets/verify_sessions_expiration_rs.js
new file mode 100644
index 00000000000..6d6b96614c7
--- /dev/null
+++ b/jstests/replsets/verify_sessions_expiration_rs.js
@@ -0,0 +1,134 @@
+// Tests valid coordination of the expiration and vivification of documents between the
+// config.system.sessions collection and the logical session cache.
+//
+// 1. Sessions should be removed from the logical session cache when they expire from
+//    the config.system.sessions collection.
+// 2. getMores run on open cursors should update the lastUse field on documents in the
+//    config.system.sessions collection, prolonging the time for expiration on said document
+//    and corresponding session.
+// 3. Open cursors that are not currently in use should be killed when their corresponding sessions
+//    expire from the config.system.sessions collection.
+// 4. Currently running operations corresponding to a session should prevent said session from
+//    expiring from the config.system.sessions collection. If the expiration date has been reached
+//    during a currently running operation, the logical session cache should vivify the session and
+//    replace it in the config.system.sessions collection.
+
+(function() {
+    "use strict";
+
+    // This test makes assertions about the number of logical session records.
+    TestData.disableImplicitSessions = true;
+
+    load("jstests/libs/pin_getmore_cursor.js");  // For "withPinnedCursor".
+
+    const refresh = {refreshLogicalSessionCacheNow: 1};
+    const startSession = {startSession: 1};
+    const failPointName = "keepCursorPinnedDuringGetMore";
+    const failPointReachedMessage = "getMore - keepCursorPinnedDuringGetMore fail point enabled";
+
+    function refreshSessionsAndVerifyCount(config, expectedCount) {
+        config.runCommand(refresh);
+        assert.eq(config.system.sessions.count(), expectedCount);
+    }
+
+    function getSessions(config) {
+        return config.system.sessions.aggregate([{'$listSessions': {allUsers: true}}]).toArray();
+    }
+
+    const dbName = "test";
+    const testCollName = "verify_sessions_find_get_more";
+
+    let replTest = new ReplSetTest({name: 'refresh', nodes: 2});
+    replTest.startSet();
+    replTest.initiate();
+
+    const primary = replTest.getPrimary();
+    replTest.awaitSecondaryNodes();
+
+    let db = primary.getDB(dbName);
+    let config = primary.getDB("config");
+
+    // 1. Verify that sessions expire from config.system.sessions after the timeout has passed.
+    for (let i = 0; i < 5; i++) {
+        let res = db.runCommand(startSession);
+        assert.commandWorked(res, "unable to start session");
+    }
+    refreshSessionsAndVerifyCount(config, 5);
+
+    // Manually delete entries in config.system.sessions to simulate TTL expiration.
+    assert.writeOK(config.system.sessions.remove({}));
+    refreshSessionsAndVerifyCount(config, 0);
+
+    // 2. Verify that getMores after finds will update the 'lastUse' field on documents in the
+    // config.system.sessions collection.
+    for (let i = 0; i < 10; i++) {
+        db[testCollName].insert({_id: i, a: i, b: 1});
+    }
+
+    let cursors = [];
+    for (let i = 0; i < 5; i++) {
+        let session = db.getMongo().startSession({});
+        assert.commandWorked(session.getDatabase("admin").runCommand({usersInfo: 1}),
+                             "initialize the session");
+        cursors.push(session.getDatabase(dbName)[testCollName].find({b: 1}).batchSize(1));
+        assert(cursors[i].hasNext());
+    }
+
+    refreshSessionsAndVerifyCount(config, 5);
+
+    let sessionsCollectionArray;
+    let lastUseValues = [];
+    for (let i = 0; i < 3; i++) {
+        for (let j = 0; j < cursors.length; j++) {
+            cursors[j].next();
+        }
+
+        refreshSessionsAndVerifyCount(config, 5);
+
+        sessionsCollectionArray = getSessions(config);
+
+        if (i == 0) {
+            for (let j = 0; j < sessionsCollectionArray.length; j++) {
+                lastUseValues.push(sessionsCollectionArray[j].lastUse);
+            }
+        } else {
+            for (let j = 0; j < sessionsCollectionArray.length; j++) {
+                assert.gt(sessionsCollectionArray[j].lastUse, lastUseValues[j]);
+                lastUseValues[j] = sessionsCollectionArray[j].lastUse;
+            }
+        }
+    }
+
+    // 3. Verify that letting sessions expire (simulated by manual deletion) will kill their
+    // cursors.
+    assert.writeOK(config.system.sessions.remove({}));
+    refreshSessionsAndVerifyCount(config, 0);
+
+    for (let i = 0; i < cursors.length; i++) {
+        assert.commandFailedWithCode(
+            db.runCommand({getMore: cursors[i]._cursor._cursorid, collection: testCollName}),
+            ErrorCodes.CursorNotFound,
+            'expected getMore to fail because the cursor was killed');
+    }
+
+    // 4. Verify that an expired session (simulated by manual deletion) that has a currently running
+    // operation will be vivified during the logical session cache refresh.
+    let pinnedCursorSession = db.getMongo().startSession();
+    let pinnedCursorDB = pinnedCursorSession.getDatabase(dbName);
+
+    withPinnedCursor({
+        conn: primary,
+        db: pinnedCursorDB,
+        assertFunction: () => {
+            config.system.sessions.remove({});
+            refreshSessionsAndVerifyCount(config, 1);
+        },
+        runGetMoreFunc: () => {
+            assert.commandWorked(db.runCommand({getMore: cursorId, collection: collName}));
+        },
+        failPointName: failPointName,
+        failPointReachedMessage: failPointReachedMessage
+    });
+
+    replTest.stopSet();
+})();
diff --git a/jstests/sharding/verify_sessions_expiration_sharded.js b/jstests/sharding/verify_sessions_expiration_sharded.js
new file mode 100644
index 00000000000..70beb6350b8
--- /dev/null
+++ b/jstests/sharding/verify_sessions_expiration_sharded.js
@@ -0,0 +1,146 @@
+// Tests valid coordination of the expiration and vivification of documents between the
+// config.system.sessions collection and the logical session cache.
+//
+// 1. Sessions should be removed from the logical session cache when they expire from
+//    the config.system.sessions collection.
+// 2. getMores run on open cursors should update the lastUse field on documents in the
+//    config.system.sessions collection, prolonging the time for expiration on said document
+//    and corresponding session.
+// 3. Open cursors that are not currently in use should be killed when their corresponding sessions
+//    expire from the config.system.sessions collection.
+// 4. Currently running operations corresponding to a session should prevent said session from
+//    expiring from the config.system.sessions collection. If the expiration date has been reached
+//    during a currently running operation, the logical session cache should vivify the session and
+//    replace it in the config.system.sessions collection.
+//
+//    @tags: [requires_find_command]
+
+(function() {
+    "use strict";
+
+    // This test makes assertions about the number of logical session records.
+    TestData.disableImplicitSessions = true;
+
+    load("jstests/libs/pin_getmore_cursor.js");  // For "withPinnedCursor".
+
+    const refresh = {refreshLogicalSessionCacheNow: 1};
+    const startSession = {startSession: 1};
+    const failPointName = "keepCursorPinnedDuringGetMore";
+    const failpointReachedMessage = "getMore - keepCursorPinnedDuringGetMore fail point enabled";
+
+    function refreshSessionsAndVerifyCount(config, expectedCount) {
+        config.runCommand(refresh);
+        assert.eq(config.system.sessions.count(), expectedCount);
+    }
+
+    function verifyOpenCursorCount(db, expectedCount) {
+        assert.eq(db.serverStatus().metrics.cursor.open.total, expectedCount);
+    }
+
+    function getSessions(config) {
+        return config.system.sessions.aggregate([{'$listSessions': {allUsers: true}}]).toArray();
+    }
+
+    const dbName = "test";
+    const testCollName = "verify_sessions_find_get_more";
+
+    let shardingTest = new ShardingTest({
+        shards: 1,
+    });
+
+    let mongos = shardingTest.s;
+    let db = mongos.getDB(dbName);
+    let config = mongos.getDB("config");
+
+    // 1. Verify that sessions expire from config.system.sessions after the timeout has passed.
+    for (let i = 0; i < 5; i++) {
+        let res = db.runCommand(startSession);
+        assert.commandWorked(res, "unable to start session");
+    }
+    refreshSessionsAndVerifyCount(config, 5);
+
+    // Manually delete entries in config.system.sessions to simulate TTL expiration.
+    assert.writeOK(config.system.sessions.remove({}));
+    refreshSessionsAndVerifyCount(config, 0);
+
+    // 2. Verify that getMores after finds will update the 'lastUse' field on documents in the
+    // config.system.sessions collection.
+    for (let i = 0; i < 10; i++) {
+        db[testCollName].insert({_id: i, a: i, b: 1});
+    }
+
+    let cursors = [];
+    for (let i = 0; i < 5; i++) {
+        let session = mongos.startSession({});
+        assert.commandWorked(session.getDatabase("admin").runCommand({usersInfo: 1}),
+                             "initialize the session");
+        cursors.push(session.getDatabase(dbName)[testCollName].find({b: 1}).batchSize(1));
+        assert(cursors[i].hasNext());
+    }
+
+    refreshSessionsAndVerifyCount(config, 5);
+    verifyOpenCursorCount(config, 5);
+
+    let sessionsCollectionArray;
+    let lastUseValues = [];
+    for (let i = 0; i < 3; i++) {
+        for (let j = 0; j < cursors.length; j++) {
+            cursors[j].next();
+        }
+
+        refreshSessionsAndVerifyCount(config, 5);
+        verifyOpenCursorCount(config, 5);
+
+        sessionsCollectionArray = getSessions(config);
+
+        if (i == 0) {
+            for (let j = 0; j < sessionsCollectionArray.length; j++) {
+                lastUseValues.push(sessionsCollectionArray[j].lastUse);
+            }
+        } else {
+            for (let j = 0; j < sessionsCollectionArray.length; j++) {
+                assert.gt(sessionsCollectionArray[j].lastUse, lastUseValues[j]);
+                lastUseValues[j] = sessionsCollectionArray[j].lastUse;
+            }
+        }
+    }
+
+    // 3. Verify that letting sessions expire (simulated by manual deletion) will kill their
+    // cursors.
+    assert.writeOK(config.system.sessions.remove({}));
+
+    refreshSessionsAndVerifyCount(config, 0);
+    verifyOpenCursorCount(config, 0);
+
+    for (let i = 0; i < cursors.length; i++) {
+        assert.commandFailedWithCode(
+            db.runCommand({getMore: cursors[i]._cursor._cursorid, collection: testCollName}),
+            ErrorCodes.CursorNotFound,
+            'expected getMore to fail because the cursor was killed');
+    }
+
+    // 4. Verify that an expired session (simulated by manual deletion) that has a currently
+    // running operation will be vivified during the logical session cache refresh.
+    let pinnedCursorSession = mongos.startSession();
+    let pinnedCursorDB = pinnedCursorSession.getDatabase(dbName);
+
+    withPinnedCursor({
+        conn: mongos,
+        sessionId: pinnedCursorSession,
+        db: pinnedCursorDB,
+        assertFunction: () => {
+            config.system.sessions.remove({});
+
+            verifyOpenCursorCount(config, 1);
+            refreshSessionsAndVerifyCount(config, 1);
+        },
+        runGetMoreFunc: () => {
+            assert.commandWorked(
+                db.runCommand({getMore: cursorId, collection: collName, lsid: sessionId}));
+        },
+        failPointName: failPointName,
+        failPointReachedMessage: failpointReachedMessage
+    });
+
+    shardingTest.stop();
+})();
diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp
index b9471f58c19..ce817497760 100644
--- a/src/mongo/db/commands/getmore_cmd.cpp
+++ b/src/mongo/db/commands/getmore_cmd.cpp
@@ -228,13 +228,18 @@ public:
         ClientCursor* cursor = ccPin.getValue().getCursor();
 
         // If the fail point is enabled, busy wait until it is disabled.
-        while (MONGO_FAIL_POINT(keepCursorPinnedDuringGetMore)) {
-            if (readLock) {
-                // We unlock and re-acquire the locks periodically in order to avoid deadlock (see
-                // SERVER-21997 for details).
-                sleepFor(Milliseconds(10));
-                readLock.reset();
-                readLock.emplace(opCtx, request.nss);
+        if (MONGO_FAIL_POINT(keepCursorPinnedDuringGetMore)) {
+            log() << "getMore - keepCursorPinnedDuringGetMore fail point "
+                     "enabled. Blocking until fail point is disabled.";
+            while (MONGO_FAIL_POINT(keepCursorPinnedDuringGetMore)) {
+                if (readLock) {
+                    // We unlock and re-acquire the locks periodically in order to avoid deadlock
+                    // (see
+                    // SERVER-21997 for details).
+                    sleepFor(Milliseconds(10));
+                    readLock.reset();
+                    readLock.emplace(opCtx, request.nss);
+                }
             }
         }
 
diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp
index 51a29efc195..6a8a6e22033 100644
--- a/src/mongo/s/query/cluster_find.cpp
+++ b/src/mongo/s/query/cluster_find.cpp
@@ -411,7 +411,12 @@ StatusWith<CursorResponse> ClusterFind::runGetMore(OperationContext* opCtx,
     invariant(request.cursorid == pinnedCursor.getValue().getCursorId());
 
     // If the fail point is enabled, busy wait until it is disabled.
-    while (MONGO_FAIL_POINT(keepCursorPinnedDuringGetMore)) {
+    if (MONGO_FAIL_POINT(keepCursorPinnedDuringGetMore)) {
+        log() << "getMore - keepCursorPinnedDuringGetMore fail point "
+                 "enabled. Blocking until fail point is disabled.";
+        while (MONGO_FAIL_POINT(keepCursorPinnedDuringGetMore)) {
+            sleepFor(Milliseconds(2));
+        }
     }
 
     // A user can only call getMore on their own cursor. If there were multiple users authenticated