SERVER-42388 Release the transaction lock resources and abort storage

transaction for unprepared transactions on failure to unstash the
transaction resources to opCtx.

(cherry picked from commit f5f62f877b62523205d02f810f351b06cf1e58c4)

3 files changed, 19 insertions, 9 deletions

diff --git a/jstests/noPassthrough/server_transaction_metrics_kill_sessions.js b/jstests/noPassthrough/server_transaction_metrics_kill_sessions.js
index 1c3315afb65..3b3fd3ec94b 100644
--- a/jstests/noPassthrough/server_transaction_metrics_kill_sessions.js
+++ b/jstests/noPassthrough/server_transaction_metrics_kill_sessions.js
@@ -55,13 +55,14 @@
 
     jsTest.log("Start a snapshot transaction at a time that is too old.");
     session.startTransaction({readConcern: {level: "snapshot", atClusterTime: Timestamp(1, 1)}});
-    // Operation runs unstashTransactionResources() and throws prior to onUnstash().
+    // Operation runs unstashTransactionResources() and throws prior to onUnstash(). As a result,
+    // the transaction will be implicitly aborted.
     assert.commandFailedWithCode(sessionDb.runCommand({find: collName}), ErrorCodes.SnapshotTooOld);
 
     newMetrics = assert.commandWorked(testDB.adminCommand({serverStatus: 1})).transactions;
     verifyMetricsChange(initialMetrics, newMetrics, "currentActive", 0);
-    verifyMetricsChange(initialMetrics, newMetrics, "currentInactive", 1);
-    verifyMetricsChange(initialMetrics, newMetrics, "currentOpen", 1);
+    verifyMetricsChange(initialMetrics, newMetrics, "currentInactive", 0);
+    verifyMetricsChange(initialMetrics, newMetrics, "currentOpen", 0);
 
     // Kill the session that threw exception before.
     jsTest.log("Kill session " + tojson(session.getSessionId()) + ".");
diff --git a/jstests/replsets/transactions_reaped_with_tickets_exhausted.js b/jstests/replsets/transactions_reaped_with_tickets_exhausted.js
index 3dfee76da09..7c5963c7bb6 100644
--- a/jstests/replsets/transactions_reaped_with_tickets_exhausted.js
+++ b/jstests/replsets/transactions_reaped_with_tickets_exhausted.js
@@ -1,6 +1,6 @@
 /**
  * Test ensures that exhausting the number of write tickets in the system does not prevent
- * transactions from being reaped by the expired transaction reaper.
+ * transactions from being reaped/aborted.
  *
  * @tags: [uses_transactions]
  */
@@ -19,10 +19,9 @@
             setParameter: {
                 wiredTigerConcurrentWriteTransactions: kNumWriteTickets,
 
-                // Setting a transaction lifetime of 20 seconds works fine locally because the
-                // threads which attempt to run the drop command are spawned quickly enough. This
-                // might not be the case for Evergreen hosts and may need to be tuned accordingly.
-                transactionLifetimeLimitSeconds: 20,
+                // Setting a transaction lifetime of 1 hour to make sure the transaction reaper
+                // doesn't abort the transaction.
+                transactionLifetimeLimitSeconds: 3600,
             }
         }
     });
@@ -75,7 +74,7 @@
         });
 
     // Attempting to perform another operation inside of the transaction will block and should
-    // eventually cause it to be aborted.
+    // cause it to be aborted implicity.
     assert.commandFailedWithCode(sessionDb.mycoll.insert({}), ErrorCodes.LockTimeout);
 
     for (let thread of threads) {
diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index 90f5a89225c..849a6744128 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -400,7 +400,17 @@ void invokeWithSessionCheckedOut(OperationContext* opCtx,
                     readConcernArgs.isEmpty());
         }
 
+        // Release the transaction lock resources and abort storage transaction for unprepared
+        // transactions on failure to unstash the transaction resources to opCtx. We don't want to
+        // have this error guard for beginOrContinue as it can abort the transaction for any
+        // accidental invalid statements in the transaction.
+        auto abortOnError = makeGuard(
+            [&txnParticipant, opCtx] { txnParticipant.abortTransactionIfNotPrepared(opCtx); });
+
         txnParticipant.unstashTransactionResources(opCtx, invocation->definition()->getName());
+
+        // Unstash success.
+        abortOnError.dismiss();
     }
 
     auto guard = makeGuard([&txnParticipant, opCtx] {