SERVER-62479 Reap sessions for the same retryable write atomically

2 files changed, 948 insertions, 29 deletions

diff --git a/jstests/replsets/internal_sessions_reaping.js b/jstests/replsets/internal_sessions_reaping_basic.js
index 09afcd9f67d..3069284020f 100644
--- a/jstests/replsets/internal_sessions_reaping.js
+++ b/jstests/replsets/internal_sessions_reaping_basic.js
@@ -28,19 +28,18 @@ rst.initiate();
 
 const primary = rst.getPrimary();
 
-const kDbName = "testDb";
-const kCollName = "testColl";
-
 const kConfigSessionsNs = "config.system.sessions";
 const kConfigTxnsNs = "config.transactions";
 const kImageCollNs = "config.image_collection";
 const kOplogCollNs = "local.oplog.rs";
+const sessionsColl = primary.getCollection(kConfigSessionsNs);
+const transactionsColl = primary.getCollection(kConfigTxnsNs);
+const imageColl = primary.getCollection(kImageCollNs);
+const oplogColl = primary.getCollection(kOplogCollNs);
 
-let sessionsCollOnPrimary = primary.getCollection(kConfigSessionsNs);
-let transactionsCollOnPrimary = primary.getCollection(kConfigTxnsNs);
-let imageCollOnPrimary = primary.getCollection(kImageCollNs);
-let oplogCollOnPrimary = primary.getCollection(kOplogCollNs);
-let testDB = primary.getDB(kDbName);
+const kDbName = "testDb";
+const kCollName = "testColl";
+const testDB = primary.getDB(kDbName);
 
 assert.commandWorked(testDB.createCollection(kCollName));
 assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
@@ -73,11 +72,11 @@ assert.commandWorked(testDB.runCommand({
 assert.commandWorked(testDB.adminCommand(
     {commitTransaction: 1, lsid: childLsid0, txnNumber: kInternalTxnNumber, autocommit: false}));
 numTransactionsCollEntries++;
-assert.eq(numTransactionsCollEntries, transactionsCollOnPrimary.find().itcount());
+assert.eq(numTransactionsCollEntries, transactionsColl.find().itcount());
 
 jsTest.log("Verify that the config.transactions entry for the internal transaction for " +
            "the non-retryable update did not get reaped after command returned");
-assert.eq(numTransactionsCollEntries, transactionsCollOnPrimary.find().itcount());
+assert.eq(numTransactionsCollEntries, transactionsColl.find().itcount());
 
 const parentTxnNumber1 = NumberLong(1);
 
@@ -122,7 +121,7 @@ numImageCollEntries++;
 
 jsTest.log("Verify that the config.transactions entry for the retryable internal transaction for " +
            "the update did not get reaped although there is already a new retryable write");
-assert.eq(numTransactionsCollEntries, transactionsCollOnPrimary.find().itcount());
+assert.eq(numTransactionsCollEntries, transactionsColl.find().itcount());
 
 const childLsid2 = {
     id: sessionUUID,
@@ -156,8 +155,8 @@ assert.commandWorked(testDB.runCommand({
 
 jsTest.log("Verify that the config.transactions entry for the retryable internal transaction for " +
            "the findAndModify did not get reaped although there is already a new retryable write");
-assert.eq(numTransactionsCollEntries, transactionsCollOnPrimary.find().itcount());
-assert.eq(numImageCollEntries, imageCollOnPrimary.find().itcount());
+assert.eq(numTransactionsCollEntries, transactionsColl.find().itcount());
+assert.eq(numImageCollEntries, imageColl.find().itcount());
 
 assert.eq({_id: 0, a: 0, b: 0, c: 0, d: 0, e: 0},
           testDB.getCollection(kCollName).findOne({_id: 0}));
@@ -165,9 +164,9 @@ assert.eq({_id: 1}, testDB.getCollection(kCollName).findOne({_id: 1}));
 
 assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
 
-assert.eq(1, sessionsCollOnPrimary.find({"_id.id": sessionUUID}).itcount());
-assert.eq(numTransactionsCollEntries, transactionsCollOnPrimary.find().itcount());
-assert.eq(numImageCollEntries, imageCollOnPrimary.find().itcount());
+assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount());
+assert.eq(numTransactionsCollEntries, transactionsColl.find().itcount());
+assert.eq(numImageCollEntries, imageColl.find().itcount());
 
 assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
 
@@ -175,29 +174,26 @@ jsTest.log("Verify that the config.transactions entries for internal transaction
            "reaped although they are expired since the config.system.sessions entry for the " +
            "parent session still has not been deleted");
 
-assert.eq(1, sessionsCollOnPrimary.find({"_id.id": sessionUUID}).itcount());
+assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount());
 assert.eq(numTransactionsCollEntries,
-          transactionsCollOnPrimary.find().itcount(),
-          tojson(transactionsCollOnPrimary.find().toArray()));
-assert.eq(numImageCollEntries, imageCollOnPrimary.find().itcount());
+          transactionsColl.find().itcount(),
+          tojson(transactionsColl.find().toArray()));
+assert.eq(numImageCollEntries, imageColl.find().itcount());
 
 // Remove the session doc so the parent session gets reaped when reapLogicalSessionCacheNow is run.
-assert.commandWorked(sessionsCollOnPrimary.remove({}));
+assert.commandWorked(sessionsColl.remove({}));
 assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
 
 jsTest.log("Verify that the config.transactions entries got reaped since the " +
            "config.system.sessions entry for the parent session had already been deleted");
-assert.eq(0, sessionsCollOnPrimary.find().itcount());
-assert.eq(0,
-          transactionsCollOnPrimary.find().itcount(),
-          tojson(transactionsCollOnPrimary.find().toArray()));
-assert.eq(0, imageCollOnPrimary.find().itcount());
+assert.eq(0, sessionsColl.find().itcount());
+assert.eq(0, transactionsColl.find().itcount(), tojson(transactionsColl.find().toArray()));
+assert.eq(0, imageColl.find().itcount());
 
 // Validate that writes to config.transactions do not generate oplog entries, with the exception of
 // deletions.
-assert.eq(numTransactionsCollEntries,
-          oplogCollOnPrimary.find({op: 'd', ns: kConfigTxnsNs}).itcount());
-assert.eq(0, oplogCollOnPrimary.find({op: {'$ne': 'd'}, ns: kConfigTxnsNs}).itcount());
+assert.eq(numTransactionsCollEntries, oplogColl.find({op: 'd', ns: kConfigTxnsNs}).itcount());
+assert.eq(0, oplogColl.find({op: {'$ne': 'd'}, ns: kConfigTxnsNs}).itcount());
 
 rst.stopSet();
 })();
diff --git a/jstests/replsets/internal_sessions_reaping_retryable_writes.js b/jstests/replsets/internal_sessions_reaping_retryable_writes.js
new file mode 100644
index 00000000000..1e972ee72df
--- /dev/null
+++ b/jstests/replsets/internal_sessions_reaping_retryable_writes.js
@@ -0,0 +1,923 @@
+/*
+ * Test that the logical cache reaper reaps Session/TransactionParticipant objects and the
+ * config.transactions and config.image_collection entries that correspond to the same retryable
+ * write atomically.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/parallelTester.js");
+load("jstests/libs/uuid_util.js");
+load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+// This test runs the reapLogicalSessionCacheNow command. That can lead to direct writes to the
+// config.transactions collection, which cannot be performed on a session.
+TestData.disableImplicitSessions = true;
+
+const rst = new ReplSetTest({
+    nodes: 2,
+    nodeOptions: {
+        setParameter: {
+            TransactionRecordMinimumLifetimeMinutes: 0,
+        }
+    }
+});
+rst.startSet();
+rst.initiate();
+
+const primary = rst.getPrimary();
+
+const kConfigSessionsNs = "config.system.sessions";
+const kConfigTxnsNs = "config.transactions";
+const kConfigImageNs = "config.image_collection";
+const sessionsColl = primary.getCollection(kConfigSessionsNs);
+const transactionsColl = primary.getCollection(kConfigTxnsNs);
+const imageColl = primary.getCollection(kConfigImageNs);
+
+const kDbName = "testDb";
+const kCollName = "testColl";
+const kNs = kDbName + "." + kCollName;
+const testDB = primary.getDB(kDbName);
+const testColl = testDB.getCollection(kCollName);
+
+assert.commandWorked(testDB.createCollection(kCollName));
+
+function makeSessionOptsForTest() {
+    const sessionUUID = UUID();
+    const parentLsid = {id: sessionUUID};
+    const parentTxnNumber = NumberLong(35);
+    const childLsidForRetryableWrite = {
+        id: sessionUUID,
+        txnNumber: parentTxnNumber,
+        txnUUID: UUID()
+    };
+    const childLsidForPrevRetryableWrite = {
+        id: sessionUUID,
+        txnNumber: NumberLong(parentTxnNumber.valueOf() - 1),
+        txnUUID: UUID()
+    };
+    const childLsidForNonRetryableWrite = {id: sessionUUID, txnUUID: UUID()};
+    const childTxnNumber = NumberLong(0);
+    return {
+        sessionUUID,
+        parentLsid,
+        parentTxnNumber,
+        childLsidForRetryableWrite,
+        childLsidForPrevRetryableWrite,
+        childLsidForNonRetryableWrite,
+        childTxnNumber,
+    };
+}
+
+function assertNumSessionsCollEntries(sessionOpts, expectedNum) {
+    const filter = {"_id.id": sessionOpts.parentLsid.id};
+    assert.eq(expectedNum,
+              sessionsColl.find(filter).itcount(),
+              tojson(sessionsColl.find(filter).toArray()));
+}
+
+function assertNumTransactionsCollEntries(sessionOpts, expectedNum) {
+    const filter = {"_id.id": sessionOpts.parentLsid.id};
+    assert.eq(expectedNum,
+              transactionsColl.find(filter).itcount(),
+              tojson(transactionsColl.find(filter).toArray()));
+}
+
+function assertNumImagesCollEntries(sessionOpts, expectedNum) {
+    const filter = {"_id.id": sessionOpts.parentLsid.id};
+    assert.eq(
+        expectedNum, imageColl.find(filter).itcount(), tojson(imageColl.find(filter).toArray()));
+}
+
+function assertNumEntries(
+    sessionOpts, {numSessionsCollEntries, numTransactionsCollEntries, numImageCollEntries}) {
+    assertNumSessionsCollEntries(sessionOpts, numSessionsCollEntries);
+    assertNumTransactionsCollEntries(sessionOpts, numTransactionsCollEntries);
+    assertNumImagesCollEntries(sessionOpts, numImageCollEntries);
+}
+
+// Test reaping when neither the external session nor the internal sessions are checked out.
+
+{
+    jsTest.log("Test reaping when there is an in-progress retryable-write internal transaction");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(0),
+    }));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.childLsidForRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(1),
+    }));
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the retryable write in the external session do not get
+    // reaped since there is an in-progress internal transaction for that retryable write.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    // Retry the write statement executed in the external session.
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        lsid: sessionOpts.childLsidForRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        autocommit: false,
+        stmtId: NumberInt(0),
+    }));
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForRetryableWrite, sessionOpts.childTxnNumber)));
+
+    // Verify that the retried write statement did not re-execute.
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+{
+    jsTest.log("Test reaping when there is an in-progress and a committed retryable-write " +
+               "internal transaction");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(0),
+    }));
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.childLsidForRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(1),
+    }));
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForRetryableWrite, sessionOpts.childTxnNumber)));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 2, numImageCollEntries: 1});
+
+    const runInternalTxn =
+        (primaryHost, parentLsidUUIDString, parentTxnNumber, dbName, collName) => {
+            load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+            const primary = new Mongo(primaryHost);
+            const testDB = primary.getDB(dbName);
+
+            const childLsid = {
+                id: UUID(parentLsidUUIDString),
+                txnNumber: NumberLong(parentTxnNumber),
+                txnUUID: UUID()
+            };
+            const childTxnNumber = NumberLong(0);
+
+            assert.commandWorked(testDB.runCommand({
+                insert: collName,
+                documents: [{x: 2}],
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                startTransaction: true,
+                autocommit: false,
+                stmtId: NumberInt(2),
+            }));
+
+            // Retry the write statement executed in the external session.
+            assert.commandWorked(testDB.runCommand({
+                findAndModify: collName,
+                query: {x: 0},
+                update: {$inc: {y: 1}},
+                new: true,
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                autocommit: false,
+                stmtId: NumberInt(0),
+            }));
+
+            // Retry the write statement executed in the committed internal transaction.
+            assert.commandWorked(testDB.runCommand({
+                insert: collName,
+                documents: [{x: 1}],
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                autocommit: false,
+                stmtId: NumberInt(1),
+            }));
+
+            assert.commandWorked(
+                primary.adminCommand(makeCommitTransactionCmdObj(childLsid, childTxnNumber)));
+        };
+
+    // Start another internal transaction in a separate thread, and make it hang right after it
+    // finishes executing the first statement.
+    const fp = configureFailPoint(primary, "waitAfterCommandFinishesExecution", {ns: kNs});
+    const internalTxnThread = new Thread(runInternalTxn,
+                                         primary.host,
+                                         extractUUIDFromObject(sessionOpts.sessionUUID),
+                                         sessionOpts.parentTxnNumber.valueOf(),
+                                         kDbName,
+                                         kCollName);
+    internalTxnThread.start();
+    fp.wait();
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the retryable write in the external session and the
+    // config.transactions for the committed internal transaction for that retryable write do not
+    // get reaped since there is an in-progress internal transaction for the same retryable write.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 2, numImageCollEntries: 1});
+
+    fp.off();
+    internalTxnThread.join();
+
+    // Verify that the retried write statements did not re-execute.
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 2}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+{
+    jsTest.log(
+        "Test reaping when there is an in-progress internal transaction for the current retryable" +
+        " write and a committed internal transaction for a previous retryable write");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.childLsidForPrevRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(0),
+    }));
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForPrevRetryableWrite, sessionOpts.childTxnNumber)));
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(1),
+    }));
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 2}],
+        lsid: sessionOpts.childLsidForRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(2),
+    }));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 2, numImageCollEntries: 1});
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the previous write do get reaped.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 1, numImageCollEntries: 0});
+
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForRetryableWrite, sessionOpts.childTxnNumber)));
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(1),
+    }));
+
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 2}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+{
+    jsTest.log("Test reaping there is an in-progress transaction in the external session and a " +
+               "committed internal transaction for a previous retryable write");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.childLsidForPrevRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(0),
+    }));
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForPrevRetryableWrite, sessionOpts.childTxnNumber)));
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+    }));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the previous write do get reaped.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(primary.adminCommand(
+        makeCommitTransactionCmdObj(sessionOpts.parentLsid, sessionOpts.parentTxnNumber)));
+
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+{
+    jsTest.log(
+        "Test reaping when there is an in-progress non retryable-write internal transaction " +
+        "and a committed retryable-write internal transaction");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.childLsidForPrevRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(0),
+    }));
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForPrevRetryableWrite, sessionOpts.childTxnNumber)));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.childLsidForNonRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(1),
+    }));
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the retryable write in external session do not get reaped
+    // since there has not been a retryble write or transaction with a higher txnNumber in the
+    // logical session.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForNonRetryableWrite, sessionOpts.childTxnNumber)));
+
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+{
+    jsTest.log("Test reaping when there is an in-progress transaction in the external session " +
+               "and a committed non retryable-write internal transaction");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.childLsidForNonRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(0),
+    }));
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForNonRetryableWrite, sessionOpts.childTxnNumber)));
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+    }));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 1, numImageCollEntries: 0});
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry for
+    // the committed non retryable-write internal transaction does get reaped since it is unrelated
+    // to the in-progress transaction in the external session.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(primary.adminCommand(
+        makeCommitTransactionCmdObj(sessionOpts.parentLsid, sessionOpts.parentTxnNumber)));
+
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+// Test reaping when there is a checked out internal session.
+
+{
+    jsTest.log("Test reaping when there is a checked out retryable-write internal session with " +
+               "an in-progress transaction");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(0),
+    }));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    const runInternalTxn =
+        (primaryHost, parentLsidUUIDString, parentTxnNumber, dbName, collName) => {
+            load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+            const primary = new Mongo(primaryHost);
+            const testDB = primary.getDB(dbName);
+
+            const childLsid = {
+                id: UUID(parentLsidUUIDString),
+                txnNumber: NumberLong(parentTxnNumber),
+                txnUUID: UUID()
+            };
+            const childTxnNumber = NumberLong(0);
+
+            assert.commandWorked(testDB.runCommand({
+                insert: collName,
+                documents: [{x: 1}],
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                startTransaction: true,
+                autocommit: false,
+                stmtId: NumberInt(1),
+            }));
+
+            // Retry the write statement executed in the external session.
+            assert.commandWorked(testDB.runCommand({
+                findAndModify: collName,
+                query: {x: 0},
+                update: {$inc: {y: 1}},
+                new: true,
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                autocommit: false,
+                stmtId: NumberInt(0),
+            }));
+
+            assert.commandWorked(
+                primary.adminCommand(makeCommitTransactionCmdObj(childLsid, childTxnNumber)));
+        };
+
+    const fp = configureFailPoint(primary, "hangAfterSessionCheckOut", {}, {skip: 1});
+    const internalTxnThread = new Thread(runInternalTxn,
+                                         primary.host,
+                                         extractUUIDFromObject(sessionOpts.sessionUUID),
+                                         sessionOpts.parentTxnNumber.valueOf(),
+                                         kDbName,
+                                         kCollName);
+    internalTxnThread.start();
+    fp.wait();
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the retryable write in the external session do not get
+    // reaped.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    fp.off();
+    internalTxnThread.join();
+
+    // Verify that the retried write statement did not re-execute.
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+{
+    jsTest.log("Test reaping when there is a checked out retryable-write internal session " +
+               "without an in-progress transaction");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(0),
+    }));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    const runInternalTxn =
+        (primaryHost, parentLsidUUIDString, parentTxnNumber, dbName, collName) => {
+            load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+            const primary = new Mongo(primaryHost);
+            const testDB = primary.getDB(dbName);
+
+            const childLsid = {
+                id: UUID(parentLsidUUIDString),
+                txnNumber: NumberLong(parentTxnNumber),
+                txnUUID: UUID()
+            };
+            const childTxnNumber = NumberLong(0);
+
+            assert.commandWorked(testDB.runCommand({
+                findAndModify: collName,
+                query: {x: 0},
+                update: {$inc: {y: 1}},
+                new: true,
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                startTransaction: true,
+                autocommit: false,
+                stmtId: NumberInt(0),
+            }));
+            assert.commandWorked(
+                primary.adminCommand(makeCommitTransactionCmdObj(childLsid, childTxnNumber)));
+        };
+
+    const fp = configureFailPoint(primary, "hangAfterSessionCheckOut");
+    const internalTxnThread = new Thread(runInternalTxn,
+                                         primary.host,
+                                         extractUUIDFromObject(sessionOpts.sessionUUID),
+                                         sessionOpts.parentTxnNumber.valueOf(),
+                                         kDbName,
+                                         kCollName);
+    internalTxnThread.start();
+    fp.wait();
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the retryable write in the external session do not get
+    // reaped.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    fp.off();
+    internalTxnThread.join();
+
+    // Verify that the retried write statement did not re-execute.
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+{
+    jsTest.log("Test reaping when there are a checked out retryable-write internal session with " +
+               "an in-progress transaction and an unchecked out retryable-write internal " +
+               "session for the same retryable write with a committed transaction");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(0),
+    }));
+
+    assert.commandWorked(testDB.runCommand({
+        insert: kCollName,
+        documents: [{x: 1}],
+        lsid: sessionOpts.childLsidForRetryableWrite,
+        txnNumber: sessionOpts.childTxnNumber,
+        startTransaction: true,
+        autocommit: false,
+        stmtId: NumberInt(1),
+    }));
+    assert.commandWorked(primary.adminCommand(makeCommitTransactionCmdObj(
+        sessionOpts.childLsidForRetryableWrite, sessionOpts.childTxnNumber)));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 2, numImageCollEntries: 1});
+
+    const runInternalTxn =
+        (primaryHost, parentLsidUUIDString, parentTxnNumber, dbName, collName) => {
+            load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+            const primary = new Mongo(primaryHost);
+            const testDB = primary.getDB(dbName);
+
+            const childLsid = {
+                id: UUID(parentLsidUUIDString),
+                txnNumber: NumberLong(parentTxnNumber),
+                txnUUID: UUID()
+            };
+            const childTxnNumber = NumberLong(0);
+
+            assert.commandWorked(testDB.runCommand({
+                insert: collName,
+                documents: [{x: 2}],
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                startTransaction: true,
+                autocommit: false,
+                stmtId: NumberInt(2),
+            }));
+
+            // Retry the write statement executed in the external session.
+            assert.commandWorked(testDB.runCommand({
+                findAndModify: collName,
+                query: {x: 0},
+                update: {$inc: {y: 1}},
+                new: true,
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                autocommit: false,
+                stmtId: NumberInt(0),
+            }));
+
+            // Retry the write statement executed in the committed internal transaction.
+            assert.commandWorked(testDB.runCommand({
+                insert: collName,
+                documents: [{x: 1}],
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                autocommit: false,
+                stmtId: NumberInt(1),
+            }));
+
+            assert.commandWorked(
+                primary.adminCommand(makeCommitTransactionCmdObj(childLsid, childTxnNumber)));
+        };
+
+    // Start another internal transaction in a separate thread, and make it hang right after it
+    // finishes executing the first statement.
+    const fp = configureFailPoint(primary, "hangInsertBeforeWrite", {ns: kNs});
+    const internalTxnThread = new Thread(runInternalTxn,
+                                         primary.host,
+                                         extractUUIDFromObject(sessionOpts.sessionUUID),
+                                         sessionOpts.parentTxnNumber.valueOf(),
+                                         kDbName,
+                                         kCollName);
+    internalTxnThread.start();
+    fp.wait();
+
+    // Force the logical session cache to reap, and verify that the config.transactions and
+    // config.image_collection entry for the retryable write in the external session and for the
+    // committed internal transaction for that retryable write do not get reaped since there is an
+    // in-progress internal transaction for the same retryable write.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 2, numImageCollEntries: 1});
+
+    fp.off();
+    internalTxnThread.join();
+
+    // Verify that the retried write statements did not re-execute.
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 1}).itcount(), 1);
+    assert.eq(testColl.find({x: 2}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+// Test reaping when an internal session is about to be checked out.
+
+{
+    jsTest.log("Test reaping when a retryable-write internal session is about to be checked out");
+    const sessionOpts = makeSessionOptsForTest();
+
+    assert.commandWorked(testColl.insert([{x: 0, y: 0}]));
+    assert.commandWorked(testDB.runCommand({
+        findAndModify: kCollName,
+        query: {x: 0},
+        update: {$inc: {y: 1}},
+        new: true,
+        lsid: sessionOpts.parentLsid,
+        txnNumber: sessionOpts.parentTxnNumber,
+        stmtId: NumberInt(0),
+    }));
+
+    assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 1, numTransactionsCollEntries: 1, numImageCollEntries: 1});
+
+    const runInternalTxn =
+        (primaryHost, parentLsidUUIDString, parentTxnNumber, dbName, collName) => {
+            load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+            const primary = new Mongo(primaryHost);
+            const testDB = primary.getDB(dbName);
+
+            const childLsid = {
+                id: UUID(parentLsidUUIDString),
+                txnNumber: NumberLong(parentTxnNumber),
+                txnUUID: UUID()
+            };
+            const childTxnNumber = NumberLong(0);
+
+            // Retry the statement executed in the external session.
+            assert.commandWorked(testDB.runCommand({
+                insert: collName,
+                documents: [{y: 0}],
+                lsid: childLsid,
+                txnNumber: childTxnNumber,
+                startTransaction: true,
+                autocommit: false,
+                stmtId: NumberInt(0),
+            }));
+            assert.commandWorked(
+                testDB.adminCommand(makeCommitTransactionCmdObj(childLsid, childTxnNumber)));
+        };
+
+    const fp = configureFailPoint(primary, "hangBeforeSessionCheckOut");
+    const internalTxnThread = new Thread(runInternalTxn,
+                                         primary.host,
+                                         extractUUIDFromObject(sessionOpts.sessionUUID),
+                                         sessionOpts.parentTxnNumber.valueOf(),
+                                         kDbName,
+                                         kCollName);
+    internalTxnThread.start();
+    fp.wait();
+
+    // Force the logical session cache to reap, and verify that the config.transactions entry and
+    // config.image_collection entry for the retryable write in the external session do get reaped.
+    assert.commandWorked(sessionsColl.remove({"_id.id": sessionOpts.sessionUUID}));
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    // Verify that the internal transaction did not get interrupted but that the retried write
+    // statement re-execute, i.e. retryablity is violated because the retry occurs after the session
+    // got reaped.
+    fp.off();
+    internalTxnThread.join();
+
+    assert.eq(testColl.find({x: 0, y: 1}).itcount(), 1);
+
+    assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1}));
+    assertNumEntries(
+        sessionOpts,
+        {numSessionsCollEntries: 0, numTransactionsCollEntries: 0, numImageCollEntries: 0});
+
+    assert.commandWorked(testColl.remove({}));
+}
+
+rst.stopSet();
+})();