SERVER-37210 Mongos should implicitly abort transactions on unhandled errors

6 files changed, 123 insertions, 11 deletions

diff --git a/jstests/sharding/libs/sharded_transactions_helpers.js b/jstests/sharding/libs/sharded_transactions_helpers.js
index 422bdac250d..d1b4a82bce0 100644
--- a/jstests/sharding/libs/sharded_transactions_helpers.js
+++ b/jstests/sharding/libs/sharded_transactions_helpers.js
@@ -19,3 +19,22 @@ function unsetFailCommandOnEachShard(st, numShards) {
             shardConn.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
     }
 }
+
+function assertNoSuchTransactionOnAllShards(st, lsid, txnNumber) {
+    st._rs.forEach(function(rs) {
+        assertNoSuchTransactionOnConn(rs.test.getPrimary(), lsid, txnNumber);
+    });
+}
+
+function assertNoSuchTransactionOnConn(conn, lsid, txnNumber) {
+    assert.commandFailedWithCode(conn.getDB("foo").runCommand({
+        find: "bar",
+        lsid: lsid,
+        txnNumber: NumberLong(txnNumber),
+        autocommit: false,
+    }),
+                                 ErrorCodes.NoSuchTransaction,
+                                 "expected there to be no active transaction on shard, lsid: " +
+                                     tojson(lsid) + ", txnNumber: " + tojson(txnNumber) +
+                                     ", connection: " + tojson(conn));
+}
diff --git a/jstests/sharding/transactions_implicit_abort.js b/jstests/sharding/transactions_implicit_abort.js
new file mode 100644
index 00000000000..4b9b7f48515
--- /dev/null
+++ b/jstests/sharding/transactions_implicit_abort.js
@@ -0,0 +1,61 @@
+// Verifies mongos will implicitly abort a transaction on all involved shards on a transaction fatal
+// error.
+//
+// @tags: [requires_sharding, uses_transactions, uses_multi_shard_transaction]
+(function() {
+    "use strict";
+
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+    const dbName = "test";
+    const collName = "foo";
+    const ns = dbName + '.' + collName;
+
+    const st = new ShardingTest({shards: 2, mongos: 1, config: 1});
+
+    // Set up a sharded collection with one chunk on each shard.
+
+    assert.writeOK(st.s.getDB(dbName)[collName].insert({_id: -1}, {writeConcern: {w: "majority"}}));
+    assert.writeOK(st.s.getDB(dbName)[collName].insert({_id: 1}, {writeConcern: {w: "majority"}}));
+
+    assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+    st.ensurePrimaryShard(dbName, st.shard0.shardName);
+
+    assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {_id: 1}}));
+    assert.commandWorked(st.s.adminCommand({split: ns, middle: {_id: 0}}));
+    assert.commandWorked(
+        st.s.adminCommand({moveChunk: ns, find: {_id: 1}, to: st.shard1.shardName}));
+
+    const session = st.s.startSession();
+    const sessionDB = session.getDatabase(dbName);
+
+    //
+    // An unhandled error during a transaction should try to abort it on all participants.
+    //
+
+    session.startTransaction();
+
+    // Targets Shard0 successfully.
+    assert.commandWorked(sessionDB.runCommand({find: collName, filter: {_id: -1}}));
+
+    assert.commandWorked(st.rs1.getPrimary().adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {errorCode: ErrorCodes.InternalError, failCommands: ["find"]}
+    }));
+
+    // Targets Shard1 and encounters a transaction fatal error.
+    assert.commandFailedWithCode(sessionDB.runCommand({find: collName, filter: {_id: 1}}),
+                                 ErrorCodes.InternalError);
+
+    assert.commandWorked(
+        st.rs1.getPrimary().adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+
+    // The transaction should have been aborted on both shards.
+    assertNoSuchTransactionOnAllShards(
+        st, session.getSessionId(), session.getTxnNumber_forTesting());
+    assert.commandFailedWithCode(session.abortTransaction_forTesting(),
+                                 ErrorCodes.NoSuchTransaction);
+
+    st.stop();
+})();
diff --git a/jstests/sharding/transactions_snapshot_errors_first_statement.js b/jstests/sharding/transactions_snapshot_errors_first_statement.js
index 7d8e3dabeb2..67516b024e6 100644
--- a/jstests/sharding/transactions_snapshot_errors_first_statement.js
+++ b/jstests/sharding/transactions_snapshot_errors_first_statement.js
@@ -114,9 +114,13 @@
                                                      ErrorCodes.NoSuchTransaction);
             assert.eq(res.errorLabels, ["TransientTransactionError"]);
 
-            session.abortTransaction();
-
             unsetFailCommandOnEachShard(st, numShardsToError);
+
+            assertNoSuchTransactionOnAllShards(
+                st, session.getSessionId(), session.getTxnNumber_forTesting());
+
+            assert.commandFailedWithCode(session.abortTransaction_forTesting(),
+                                         ErrorCodes.NoSuchTransaction);
         }
     }
 
@@ -162,7 +166,7 @@
 
     // Test only one shard throwing the error when more than one are targeted.
     for (let errorCode of kSnapshotErrors) {
-        runTest(st, collName, 1, errorCode, 2);
+        runTest(st, collName, 1, errorCode, true);
     }
 
     st.stop();
diff --git a/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js b/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
index 28a6e861fee..1ff45f75302 100644
--- a/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
+++ b/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
@@ -59,7 +59,10 @@
                                                      ErrorCodes.NoSuchTransaction);
             assert.eq(res.errorLabels, ["TransientTransactionError"]);
 
-            session.abortTransaction();
+            assertNoSuchTransactionOnAllShards(
+                st, session.getSessionId(), session.getTxnNumber_forTesting());
+            assert.commandFailedWithCode(session.abortTransaction_forTesting(),
+                                         ErrorCodes.NoSuchTransaction);
         }
     }
 
diff --git a/jstests/sharding/transactions_stale_database_version_errors.js b/jstests/sharding/transactions_stale_database_version_errors.js
index 6c3329c5400..a4a2db4d294 100644
--- a/jstests/sharding/transactions_stale_database_version_errors.js
+++ b/jstests/sharding/transactions_stale_database_version_errors.js
@@ -4,6 +4,8 @@
 (function() {
     "use strict";
 
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
     const dbName = "test";
     const collName = "foo";
 
@@ -50,7 +52,10 @@
         ErrorCodes.NoSuchTransaction);
     assert.eq(res.errorLabels, ["TransientTransactionError"]);
 
-    session.abortTransaction();
+    assertNoSuchTransactionOnAllShards(
+        st, session.getSessionId(), session.getTxnNumber_forTesting());
+    assert.commandFailedWithCode(session.abortTransaction_forTesting(),
+                                 ErrorCodes.NoSuchTransaction);
 
     //
     // Stale database version on first command to a new shard should succeed.
@@ -91,6 +96,7 @@
     //
 
     st.ensurePrimaryShard(dbName, st.shard0.shardName);
+    st.ensurePrimaryShard(otherDbName, st.shard1.shardName);
 
     // Disable database metadata refreshes on the stale shard so it will indefinitely return a stale
     // version error.
@@ -99,13 +105,20 @@
 
     session.startTransaction();
 
+    // Target Shard1, to verify the transaction on it is implicitly aborted later.
+    assert.commandWorked(sessionOtherDB.runCommand({find: otherCollName}));
+
     // Target the first database which is on Shard0. The shard is stale and won't refresh its
     // metadata, so mongos should exhaust its retries and implicitly abort the transaction.
     assert.commandFailedWithCode(
         sessionDB.runCommand({distinct: collName, key: "_id", query: {_id: 0}}),
         ErrorCodes.NoSuchTransaction);
 
-    session.abortTransaction();
+    // Verify all shards aborted the transaction.
+    assertNoSuchTransactionOnAllShards(
+        st, session.getSessionId(), session.getTxnNumber_forTesting());
+    assert.commandFailedWithCode(session.abortTransaction_forTesting(),
+                                 ErrorCodes.NoSuchTransaction);
 
     assert.commandWorked(st.rs0.getPrimary().adminCommand(
         {configureFailPoint: "skipDatabaseVersionMetadataRefresh", mode: "off"}));
diff --git a/jstests/sharding/transactions_stale_shard_version_errors.js b/jstests/sharding/transactions_stale_shard_version_errors.js
index 7e796ca80a0..0db7eb517ea 100644
--- a/jstests/sharding/transactions_stale_shard_version_errors.js
+++ b/jstests/sharding/transactions_stale_shard_version_errors.js
@@ -4,6 +4,8 @@
 (function() {
     "use strict";
 
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
     function expectChunks(st, ns, chunks) {
         for (let i = 0; i < chunks.length; i++) {
             assert.eq(chunks[i],
@@ -101,7 +103,10 @@
                                      ErrorCodes.NoSuchTransaction);
     assert.eq(res.errorLabels, ["TransientTransactionError"]);
 
-    session.abortTransaction();
+    assertNoSuchTransactionOnAllShards(
+        st, session.getSessionId(), session.getTxnNumber_forTesting());
+    assert.commandFailedWithCode(session.abortTransaction_forTesting(),
+                                 ErrorCodes.NoSuchTransaction);
 
     //
     // Stale shard version on first command to a new shard should succeed.
@@ -237,12 +242,19 @@
 
     session.startTransaction();
 
-    // Targets Shard0, which is stale and won't refresh its metadata, so mongos should exhaust its
-    // retries and implicitly abort the transaction.
-    assert.commandFailedWithCode(sessionDB.runCommand({find: collName, filter: {_id: -5}}),
+    // Target Shard2, to verify the transaction on it is aborted implicitly later.
+    assert.commandWorked(sessionDB.runCommand({find: collName, filter: {_id: 5}}));
+
+    // Targets all shards. Shard0 is stale and won't refresh its metadata, so mongos should exhaust
+    // its retries and implicitly abort the transaction.
+    assert.commandFailedWithCode(sessionDB.runCommand({find: collName}),
                                  ErrorCodes.NoSuchTransaction);
 
-    session.abortTransaction();
+    // Verify the shards that did not return an error were also aborted.
+    assertNoSuchTransactionOnAllShards(
+        st, session.getSessionId(), session.getTxnNumber_forTesting());
+    assert.commandFailedWithCode(session.abortTransaction_forTesting(),
+                                 ErrorCodes.NoSuchTransaction);
 
     assert.commandWorked(st.rs0.getPrimary().adminCommand(
         {configureFailPoint: "skipShardFilteringMetadataRefresh", mode: "off"}));