SERVER-63098 Split internal_transactions_for_retryable_findAndModify_restart.js and internal_transactions_for_retryable_findAndModify_failover.js

13 files changed, 221 insertions, 67 deletions

diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_basic.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_basic.js
new file mode 100644
index 00000000000..4462c1420f9
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_basic.js
@@ -0,0 +1,39 @@
+/*
+ * Tests that retryable internal transactions for findAndModify are retryable and other kinds of
+ * transactions for findAndModify are not retryable.
+ *
+ * @tags: [requires_fcv_51, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+
+{
+    jsTest.log("Test that non-internal transactions cannot be retried");
+    const lsid = {id: UUID()};
+    const testOptions = {expectRetryToSucceed: false};
+    transactionTest.runFindAndModifyTestsEnableImageCollection(lsid, testOptions);
+}
+
+{
+    jsTest.log("Test that non-retryable internal transactions cannot be retried");
+    const lsid = {id: UUID(), txnUUID: UUID()};
+    const testOptions = {expectRetryToSucceed: false};
+    transactionTest.runFindAndModifyTestsEnableImageCollection(lsid, testOptions);
+}
+
+{
+    jsTest.log("Test that retryable internal transactions can be retried");
+    transactionTest.runTestsForAllRetryableInternalTransactionTypes(
+        transactionTest.runFindAndModifyTestsEnableImageCollection,
+        transactionTest.TestMode.kNonRecovery);
+    transactionTest.runTestsForAllRetryableInternalTransactionTypes(
+        transactionTest.runFindAndModifyTestsDisableImageCollection,
+        transactionTest.TestMode.kNonRecovery);
+}
+
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover.js
deleted file mode 100644
index 4050074c5d4..00000000000
--- a/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover.js
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Tests that retryable internal transactions for findAndModify are retryable across failover.
- *
- * @tags: [requires_fcv_51, featureFlagInternalTransactions]
- */
-(function() {
-'use strict';
-
-load('jstests/sharding/libs/retryable_internal_transaction_test.js');
-
-const transactionTest = new RetryableInternalTransactionTest();
-transactionTest.runTestsForAllRetryableInternalTransactionTypes(
-    transactionTest.runFindAndModifyTests, transactionTest.TestMode.kFailover);
-transactionTest.stop();
-})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_prepared_transaction_with_image_collection_disabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_prepared_transaction_with_image_collection_disabled.js
new file mode 100644
index 00000000000..d15cf4da3fd
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_prepared_transaction_with_image_collection_disabled.js
@@ -0,0 +1,17 @@
+/*
+ * Tests that prepared retryable internal transactions for findAndModify on a shard with image
+ * collection disabled are retryable across failover.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllPreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsDisableImageCollection,
+    transactionTest.TestMode.kFailover);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_prepared_transaction_with_image_collection_enabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_prepared_transaction_with_image_collection_enabled.js
new file mode 100644
index 00000000000..ffbfa389d0f
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_prepared_transaction_with_image_collection_enabled.js
@@ -0,0 +1,16 @@
+/*
+ * Tests that prepared retryable internal transactions for findAndModify on a shard with image
+ * collection enabled are retryable across failover.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllPreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsEnableImageCollection, transactionTest.TestMode.kFailover);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_unprepared_transaction_with_image_collection_disabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_unprepared_transaction_with_image_collection_disabled.js
new file mode 100644
index 00000000000..8bec5dad04b
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_unprepared_transaction_with_image_collection_disabled.js
@@ -0,0 +1,17 @@
+/*
+ * Tests that unprepared retryable internal transactions for findAndModify on a shard with image
+ * collection disabled are retryable across failover.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllUnpreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsDisableImageCollection,
+    transactionTest.TestMode.kFailover);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_unprepared_transaction_with_image_collection_enabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_unprepared_transaction_with_image_collection_enabled.js
new file mode 100644
index 00000000000..110332e3b73
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_failover_unprepared_transaction_with_image_collection_enabled.js
@@ -0,0 +1,16 @@
+/*
+ * Tests that unprepared retryable internal transactions for findAndModify on a shard with image
+ * collection enabled are retryable across failover.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllUnpreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsEnableImageCollection, transactionTest.TestMode.kFailover);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart.js
deleted file mode 100644
index 002037979b4..00000000000
--- a/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart.js
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Tests that retryable internal transactions for findAndModify are retryable across restart.
- *
- * @tags: [requires_fcv_53, featureFlagInternalTransactions]
- */
-(function() {
-'use strict';
-
-load('jstests/sharding/libs/retryable_internal_transaction_test.js');
-
-const transactionTest = new RetryableInternalTransactionTest();
-transactionTest.runTestsForAllRetryableInternalTransactionTypes(
-    transactionTest.runFindAndModifyTests, transactionTest.TestMode.kRestart);
-transactionTest.stop();
-})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_prepared_transaction_with_image_collection_disabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_prepared_transaction_with_image_collection_disabled.js
new file mode 100644
index 00000000000..3e5d6473a4f
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_prepared_transaction_with_image_collection_disabled.js
@@ -0,0 +1,16 @@
+/*
+ * Tests that prepared retryable internal transactions for findAndModify on a shard with image
+ * collection disabled are retryable across restart.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllPreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsDisableImageCollection, transactionTest.TestMode.kRestart);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_prepared_transaction_with_image_collection_enabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_prepared_transaction_with_image_collection_enabled.js
new file mode 100644
index 00000000000..4bbcc55bf79
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_prepared_transaction_with_image_collection_enabled.js
@@ -0,0 +1,16 @@
+/*
+ * Tests that prepared retryable internal transactions for findAndModify on a shard with image
+ * collection enabled are retryable across restart.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllPreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsEnableImageCollection, transactionTest.TestMode.kRestart);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_unprepared_transaction_with_image_collection_disabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_unprepared_transaction_with_image_collection_disabled.js
new file mode 100644
index 00000000000..1b93e43caa6
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_unprepared_transaction_with_image_collection_disabled.js
@@ -0,0 +1,16 @@
+/*
+ * Tests that unprepared retryable internal transactions for findAndModify on a shard with image
+ * collection disabled are retryable across restart.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllUnpreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsDisableImageCollection, transactionTest.TestMode.kRestart);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_unprepared_transaction_with_image_collection_enabled.js b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_unprepared_transaction_with_image_collection_enabled.js
new file mode 100644
index 00000000000..d5c06ddddaf
--- /dev/null
+++ b/jstests/sharding/internal_transactions_for_retryable_findAndModify_restart_unprepared_transaction_with_image_collection_enabled.js
@@ -0,0 +1,16 @@
+/*
+ * Tests that unprepared retryable internal transactions for findAndModify on a shard with image
+ * collection enabled are retryable across restart.
+ *
+ * @tags: [requires_fcv_53, featureFlagInternalTransactions]
+ */
+(function() {
+'use strict';
+
+load('jstests/sharding/libs/retryable_internal_transaction_test.js');
+
+const transactionTest = new RetryableInternalTransactionTest();
+transactionTest.runTestsForAllUnpreparedRetryableInternalTransactionTypes(
+    transactionTest.runFindAndModifyTestsEnableImageCollection, transactionTest.TestMode.kRestart);
+transactionTest.stop();
+})();
diff --git a/jstests/sharding/internal_transactions_for_retryable_writes_basic.js b/jstests/sharding/internal_transactions_for_retryable_insert_update_delete_basic.js
index b816cc60c23..38d922c8107 100644
--- a/jstests/sharding/internal_transactions_for_retryable_writes_basic.js
+++ b/jstests/sharding/internal_transactions_for_retryable_insert_update_delete_basic.js
@@ -1,5 +1,6 @@
 /*
- * Tests that retryable internal transactions are retryable and that other transactions are not.
+ * Tests that retryable internal transactions for insert, update and delete are retryable and
+ * other kinds of transactions for insert, update and delete are not retryable.
  *
  * @tags: [requires_fcv_51, featureFlagInternalTransactions]
  */
@@ -15,7 +16,6 @@ const transactionTest = new RetryableInternalTransactionTest();
     const lsid = {id: UUID()};
     const testOptions = {expectRetryToSucceed: false};
     transactionTest.runInsertUpdateDeleteTests(lsid, testOptions);
-    transactionTest.runFindAndModifyTests(lsid, testOptions);
 }
 
 {
@@ -23,15 +23,12 @@ const transactionTest = new RetryableInternalTransactionTest();
     const lsid = {id: UUID(), txnUUID: UUID()};
     const testOptions = {expectRetryToSucceed: false};
     transactionTest.runInsertUpdateDeleteTests(lsid, testOptions);
-    transactionTest.runFindAndModifyTests(lsid, testOptions);
 }
 
 {
     jsTest.log("Test that retryable internal transactions can be retried");
     transactionTest.runTestsForAllRetryableInternalTransactionTypes(
-        transactionTest.runInsertUpdateDeleteTests);
-    transactionTest.runTestsForAllRetryableInternalTransactionTypes(
-        transactionTest.runFindAndModifyTests);
+        transactionTest.runInsertUpdateDeleteTests, transactionTest.TestMode.kNonRecovery);
 }
 
 transactionTest.stop();
diff --git a/jstests/sharding/libs/retryable_internal_transaction_test.js b/jstests/sharding/libs/retryable_internal_transaction_test.js
index 54658f7d52a..2d3ac9d4863 100644
--- a/jstests/sharding/libs/retryable_internal_transaction_test.js
+++ b/jstests/sharding/libs/retryable_internal_transaction_test.js
@@ -39,6 +39,10 @@ function RetryableInternalTransactionTest() {
     const st = new ShardingTest({shards: 1, rs: {nodes: 2, oplogSize: 256}});
 
     const kTestMode = {kNonRecovery: 1, kRestart: 2, kFailover: 3, kRollback: 4};
+
+    // Used when testing large transactions (i.e. in 'testRetryLargeTxn') for specifying which
+    // applyOps oplog entry should contain the entry for retryable write being tested.
+    // 'testRetryLargeTxn' runs a large transaction with three applyOps oplog entries.
     const kOplogEntryLocation = {kFirst: 1, kMiddle: 2, kLast: 3};
 
     // For creating documents that will result in large transactions.
@@ -55,6 +59,11 @@ function RetryableInternalTransactionTest() {
         return {id: UUID(), txnNumber: NumberLong(0), txnUUID: UUID()};
     }
 
+    const getRandomOplogEntryLocation = function() {
+        const locations = Object.values(kOplogEntryLocation);
+        return locations[Math.floor(Math.random() * locations.length)];
+    };
+
     function getTransactionState(lsid, txnNumber) {
         return {
             oplogEntries: getOplogEntriesForTxnWithRetries(st.rs0, lsid, txnNumber),
@@ -91,9 +100,8 @@ function RetryableInternalTransactionTest() {
         if (isPreparedTxn && !isRetry) {
             const shard0Primary = st.rs0.getPrimary();
             const prepareCmdObj = makePrepareTransactionCmdObj(lsid, txnNumber);
-            const isPreparedTxnRes =
-                assert.commandWorked(shard0Primary.adminCommand(prepareCmdObj));
-            commitCmdObj.commitTimestamp = isPreparedTxnRes.prepareTimestamp;
+            const prepareRes = assert.commandWorked(shard0Primary.adminCommand(prepareCmdObj));
+            commitCmdObj.commitTimestamp = prepareRes.prepareTimestamp;
             assert.commandWorked(shard0Primary.adminCommand(commitCmdObj));
         }
         assert.commandWorked(mongosTestDB.adminCommand(commitCmdObj));
@@ -434,6 +442,9 @@ function RetryableInternalTransactionTest() {
         testOptions.lastUsedTxnNumber =
             testOptions.lastUsedTxnNumber ? testOptions.lastUsedTxnNumber : 0;
         testOptions.txnOptions = testOptions.txnOptions ? testOptions.txnOptions : {};
+        if (testOptions.txnOptions.isLargeTxn) {
+            testOptions.txnOptions.oplogEntryLocation = getRandomOplogEntryLocation();
+        }
         jsTest.log(`Testing insert, update and delete with options: ${tojson(testOptions)}`);
 
         testRetryInserts(lsid, testOptions.lastUsedTxnNumber++, testOptions);
@@ -441,38 +452,35 @@ function RetryableInternalTransactionTest() {
         testRetryDeletes(lsid, testOptions.lastUsedTxnNumber++, testOptions);
     };
 
-    this.runFindAndModifyTests = function(lsid, testOptions) {
+    function runFindAndModifyTests(lsid, testOptions) {
         testOptions.lastUsedTxnNumber =
             testOptions.lastUsedTxnNumber ? testOptions.lastUsedTxnNumber : 0;
         testOptions.txnOptions = testOptions.txnOptions ? testOptions.txnOptions : {};
-
-        const oplogEntryLocations = testOptions.txnOptions.isLargeTxn
-            ? [kOplogEntryLocation.kFirst, kOplogEntryLocation.kMiddle, kOplogEntryLocation.kLast]
-            : [kOplogEntryLocation.kFirst];
-        for (let oplogEntryLocation in oplogEntryLocations) {
-            testOptions.txnOptions.oplogEntryLocation = oplogEntryLocation;
-            jsTest.log(`Testing findAndModify with options: ${tojson(testOptions)}`);
-
-            testOptions.enableFindAndModifyImageCollection = true;
-            testRetryFindAndModifyUpsert(lsid, testOptions.lastUsedTxnNumber++, testOptions);
-            testRetryFindAndModifyUpdateWithPreImage(
-                lsid, testOptions.lastUsedTxnNumber++, testOptions);
-            testRetryFindAndModifyUpdateWithPostImage(
-                lsid, testOptions.lastUsedTxnNumber++, testOptions);
-            testRetryFindAndModifyRemove(lsid, testOptions.lastUsedTxnNumber++, testOptions);
-
-            testOptions.enableFindAndModifyImageCollection = false;
-            testRetryFindAndModifyUpsert(lsid, testOptions.lastUsedTxnNumber++, testOptions);
-            testRetryFindAndModifyUpdateWithPreImage(
-                lsid, testOptions.lastUsedTxnNumber++, testOptions);
-            testRetryFindAndModifyUpdateWithPostImage(
-                lsid, testOptions.lastUsedTxnNumber++, testOptions);
-            testRetryFindAndModifyRemove(lsid, testOptions.lastUsedTxnNumber++, testOptions);
+        if (testOptions.txnOptions.isLargeTxn) {
+            testOptions.txnOptions.oplogEntryLocation = getRandomOplogEntryLocation();
         }
+        jsTest.log(`Testing findAndModify with options: ${tojson(testOptions)}`);
+
+        testRetryFindAndModifyUpsert(lsid, testOptions.lastUsedTxnNumber++, testOptions);
+        testRetryFindAndModifyUpdateWithPreImage(
+            lsid, testOptions.lastUsedTxnNumber++, testOptions);
+        testRetryFindAndModifyUpdateWithPostImage(
+            lsid, testOptions.lastUsedTxnNumber++, testOptions);
+        testRetryFindAndModifyRemove(lsid, testOptions.lastUsedTxnNumber++, testOptions);
+    }
+
+    this.runFindAndModifyTestsEnableImageCollection = function(lsid, testOptions) {
+        testOptions.enableFindAndModifyImageCollection = true;
+        runFindAndModifyTests(lsid, testOptions);
+    };
+
+    this.runFindAndModifyTestsDisableImageCollection = function(lsid, testOptions) {
+        testOptions.enableFindAndModifyImageCollection = false;
+        runFindAndModifyTests(lsid, testOptions);
     };
 
-    this.runTestsForAllRetryableInternalTransactionTypes = function(
-        runTestsFunc, testMode = kTestMode.kNonRecovery) {
+    this.runTestsForAllUnpreparedRetryableInternalTransactionTypes = function(runTestsFunc,
+                                                                              testMode) {
         const expectRetryToSucceed = true;
 
         runTestsFunc(makeSessionIdForRetryableInternalTransaction(), {
@@ -483,17 +491,27 @@ function RetryableInternalTransactionTest() {
 
         runTestsFunc(
             makeSessionIdForRetryableInternalTransaction(),
-            {expectRetryToSucceed, txnOptions: {isPreparedTxn: true, isLargeTxn: false}, testMode});
+            {expectRetryToSucceed, txnOptions: {isPreparedTxn: false, isLargeTxn: true}, testMode});
+    };
+
+    this.runTestsForAllPreparedRetryableInternalTransactionTypes = function(runTestsFunc,
+                                                                            testMode) {
+        const expectRetryToSucceed = true;
 
         runTestsFunc(
             makeSessionIdForRetryableInternalTransaction(),
-            {expectRetryToSucceed, txnOptions: {isPreparedTxn: false, isLargeTxn: true}, testMode});
+            {expectRetryToSucceed, txnOptions: {isPreparedTxn: true, isLargeTxn: false}, testMode});
 
         runTestsFunc(
             makeSessionIdForRetryableInternalTransaction(),
             {expectRetryToSucceed, txnOptions: {isPreparedTxn: true, isLargeTxn: true}, testMode});
     };
 
+    this.runTestsForAllRetryableInternalTransactionTypes = function(runTestsFunc, testMode) {
+        this.runTestsForAllUnpreparedRetryableInternalTransactionTypes(runTestsFunc, testMode);
+        this.runTestsForAllPreparedRetryableInternalTransactionTypes(runTestsFunc, testMode);
+    };
+
     this.stop = function() {
         st.stop();
     };