SERVER-37344 Implement recovery token for retrying a commit command on a different mongos

3 files changed, 70 insertions, 33 deletions

diff --git a/jstests/core/txns/commands_not_allowed_in_txn.js b/jstests/core/txns/commands_not_allowed_in_txn.js
index 08027394a00..d86d06e775a 100644
--- a/jstests/core/txns/commands_not_allowed_in_txn.js
+++ b/jstests/core/txns/commands_not_allowed_in_txn.js
@@ -53,13 +53,17 @@
             autocommit: false
         })),
                                      ErrorCodes.OperationNotSupportedInTransaction);
-        assert.commandFailedWithCode(sessionDb.adminCommand({
-            commitTransaction: 1,
-            txnNumber: NumberLong(txnNumber),
-            stmtId: NumberInt(1),
-            autocommit: false
-        }),
-                                     ErrorCodes.NoSuchTransaction);
+
+        // Mongos has special handling for commitTransaction to support commit recovery.
+        if (!isMongos) {
+            assert.commandFailedWithCode(sessionDb.adminCommand({
+                commitTransaction: 1,
+                txnNumber: NumberLong(txnNumber),
+                stmtId: NumberInt(1),
+                autocommit: false
+            }),
+                                         ErrorCodes.NoSuchTransaction);
+        }
 
         // Check that the command fails inside a transaction, but does not abort the transaction.
         setup();
@@ -184,14 +188,17 @@
     }),
                                  ErrorCodes.OperationNotSupportedInTransaction);
 
-    // The failed find should abort the transaction so a commit should fail.
-    assert.commandFailedWithCode(sessionDb.adminCommand({
-        commitTransaction: 1,
-        autocommit: false,
-        txnNumber: NumberLong(txnNumber),
-        stmtId: NumberInt(1),
-    }),
-                                 ErrorCodes.NoSuchTransaction);
+    // Mongos has special handling for commitTransaction to support commit recovery.
+    if (!isMongos) {
+        // The failed find should abort the transaction so a commit should fail.
+        assert.commandFailedWithCode(sessionDb.adminCommand({
+            commitTransaction: 1,
+            autocommit: false,
+            txnNumber: NumberLong(txnNumber),
+            stmtId: NumberInt(1),
+        }),
+                                     ErrorCodes.NoSuchTransaction);
+    }
 
     session.endSession();
 }());
diff --git a/jstests/core/txns/multi_statement_transaction_command_args.js b/jstests/core/txns/multi_statement_transaction_command_args.js
index 90ed7dd9a7a..abffa8ec1b5 100644
--- a/jstests/core/txns/multi_statement_transaction_command_args.js
+++ b/jstests/core/txns/multi_statement_transaction_command_args.js
@@ -7,6 +7,7 @@
 (function() {
     "use strict";
     load('jstests/libs/uuid_util.js');
+    load("jstests/libs/fixture_helpers.js");  // For FixtureHelpers.
 
     // Makes assertions on commands run without logical session ids.
     TestData.disableImplicitSessions = true;
@@ -176,14 +177,17 @@
     }),
                                  ErrorCodes.InvalidOptions);
 
-    // Committing the transaction should fail.
-    assert.commandFailedWithCode(sessionDb.adminCommand({
-        commitTransaction: 1,
-        txnNumber: NumberLong(txnNumber),
-        autocommit: false,
-        writeConcern: {w: "majority"}
-    }),
-                                 ErrorCodes.NoSuchTransaction);
+    // Mongos has special handling for commitTransaction to support commit recovery.
+    if (!FixtureHelpers.isMongos(sessionDb)) {
+        // Committing the transaction should fail.
+        assert.commandFailedWithCode(sessionDb.adminCommand({
+            commitTransaction: 1,
+            txnNumber: NumberLong(txnNumber),
+            autocommit: false,
+            writeConcern: {w: "majority"}
+        }),
+                                     ErrorCodes.NoSuchTransaction);
+    }
 
     jsTestLog("Run a non-initial transaction operation with autocommit=true");
     txnNumber++;
diff --git a/jstests/sharding/transactions_recover_decision_from_local_participant.js b/jstests/sharding/transactions_recover_decision_from_local_participant.js
index 784d4e097ae..98fc5dc7562 100644
--- a/jstests/sharding/transactions_recover_decision_from_local_participant.js
+++ b/jstests/sharding/transactions_recover_decision_from_local_participant.js
@@ -10,17 +10,17 @@
     // The test modifies config.transactions, which must be done outside of a session.
     TestData.disableImplicitSessions = true;
 
-    let st = new ShardingTest({shards: 2});
+    let st = new ShardingTest({shards: 2, mongos: 2});
 
-    assert.commandWorked(st.s.adminCommand({enableSharding: 'test'}));
+    assert.commandWorked(st.s0.adminCommand({enableSharding: 'test'}));
     st.ensurePrimaryShard('test', st.shard0.name);
-    assert.commandWorked(st.s.adminCommand({shardCollection: 'test.user', key: {x: 1}}));
-    assert.commandWorked(st.s.adminCommand({split: 'test.user', middle: {x: 0}}));
+    assert.commandWorked(st.s0.adminCommand({shardCollection: 'test.user', key: {x: 1}}));
+    assert.commandWorked(st.s0.adminCommand({split: 'test.user', middle: {x: 0}}));
     assert.commandWorked(
-        st.s.adminCommand({moveChunk: 'test.user', find: {x: 0}, to: st.shard1.name}));
+        st.s0.adminCommand({moveChunk: 'test.user', find: {x: 0}, to: st.shard1.name}));
 
     // Insert documents to prime mongos and shards with the latest sharding metadata.
-    let testDB = st.s.getDB('test');
+    let testDB = st.s0.getDB('test');
     assert.commandWorked(testDB.runCommand({insert: 'user', documents: [{x: -10}, {x: 10}]}));
 
     let coordinatorConn = st.rs0.getPrimary();
@@ -46,7 +46,8 @@
             u: {"$set": {lastTxnNumber: txnNumber}},
             upsert: true
         };
-        assert.commandWorked(testDB.runCommand({
+
+        let res = assert.commandWorked(testDB.runCommand({
             update: 'user',
             updates: [updateDocumentOnShard0, updateDocumentOnShard1],
             lsid: lsid,
@@ -54,6 +55,19 @@
             autocommit: false,
             startTransaction: true
         }));
+
+        assert.neq(null, res.recoveryToken);
+        return res.recoveryToken;
+    };
+
+    const sendCommitViaOtherMongos = function(lsid, txnNumber, recoveryToken) {
+        return st.s1.getDB('admin').runCommand({
+            commitTransaction: 1,
+            lsid: lsid,
+            txnNumber: NumberLong(txnNumber),
+            autocommit: false,
+            recoveryToken: recoveryToken
+        });
     };
 
     // TODO (SERVER-37364): Once coordinateCommit returns as soon as the decision is made durable,
@@ -69,7 +83,8 @@
         jsTest.log(
             "coordinateCommit sent after coordinator finished coordinating an abort decision.");
         ++txnNumber;
-        startNewTransactionThroughMongos();
+
+        let recoveryToken = startNewTransactionThroughMongos();
         assert.commandWorked(st.rs0.getPrimary().adminCommand({
             abortTransaction: 1,
             lsid: lsid,
@@ -86,10 +101,14 @@
         assert.commandFailedWithCode(runCoordinateCommit(txnNumber, participantList),
                                      ErrorCodes.NoSuchTransaction);
 
+        assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken),
+                                     ErrorCodes.NoSuchTransaction);
+
         jsTest.log(
             "coordinateCommit sent after coordinator finished coordinating a commit decision.");
         ++txnNumber;
-        startNewTransactionThroughMongos();
+
+        recoveryToken = startNewTransactionThroughMongos();
         assert.commandWorked(testDB.adminCommand({
             commitTransaction: 1,
             lsid: lsid,
@@ -98,6 +117,8 @@
         }));
         assert.commandWorked(runCoordinateCommit(txnNumber, participantList));
 
+        assert.commandWorked(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken));
+
         jsTest.log(
             "coordinateCommit sent for lower transaction number than last number participant saw.");
         assert.commandFailedWithCode(runCoordinateCommit(txnNumber - 1, participantList),
@@ -115,7 +136,7 @@
         jsTest.log(
             "coordinateCommit sent for higher transaction number than participant has seen.");
         ++txnNumber;
-        startNewTransactionThroughMongos();
+        recoveryToken = startNewTransactionThroughMongos();
         assert.commandFailedWithCode(runCoordinateCommit(txnNumber + 1, participantList),
                                      ErrorCodes.NoSuchTransaction);
 
@@ -130,6 +151,11 @@
             autocommit: false
         }),
                                      ErrorCodes.NoSuchTransaction);
+
+        // Previous commit already discarded the coordinator since it aborted, so we get
+        // "transaction too old" instead.
+        assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken),
+                                     ErrorCodes.TransactionTooOld);
     };
 
     // Test with a real participant list, to simulate retrying through main router.