SERVER-56188 Nested applyOps can hold onto global X lock while blocking on migration commit/abort

(cherry picked from commit 51bef6c2d9d035afed3735ea107d1e7b58392d93)

8 files changed, 178 insertions, 137 deletions

diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_causally_consistent_jscore_passthrough.yml
index c7d263c9732..3cf95987f72 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_causally_consistent_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_causally_consistent_jscore_passthrough.yml
@@ -78,6 +78,32 @@ selector:
   - jstests/core/notablescan_capped.js
   # captrunc command is not blocked during tenant migration.
   - jstests/core/capped6.js
+  # These tests run applyOps which is not supported in serverless.
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops2.js
+  - jstests/core/apply_ops_dups.js
+  - jstests/core/apply_ops_index_collation.js
+  - jstests/core/apply_ops_invalid_index_spec.js
+  - jstests/core/apply_ops_missing_field.js
+  - jstests/core/apply_ops_system_dot_views.js
+  - jstests/core/apply_ops_without_ns.js
+  - jstests/core/bypass_doc_validation.js
+  - jstests/core/collation.js
+  - jstests/core/collmod_without_uuid.js
+  - jstests/core/txns/commands_banning_txnnumber_outside_transactions.js
+  - jstests/core/txns/commands_not_allowed_in_txn.js
+  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js
+  - jstests/core/txns/statement_ids_accepted.js
+  - jstests/core/list_collections1.js
+  - jstests/core/list_collections_filter.js
+  - jstests/core/list_collections_no_views.js
+  - jstests/core/views/duplicate_ns.js
+  - jstests/core/views/view_with_invalid_dbname.js
+  - jstests/core/views/views_creation.js
+  - jstests/core/views/invalid_system_views.js
+  - jstests/core/views/views_all_commands.js
+  - jstests/core/rename_stayTemp.js
 
   #
   # Blacklists from causally_consistent_jscore_passthrough.yml
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_jscore_passthrough.yml
index 63f42a43f8e..a65cb19e6bc 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_jscore_passthrough.yml
@@ -63,6 +63,32 @@ selector:
   - jstests/core/invalidated_legacy_cursors.js
   # TODO (SERVER-52727): Synchronize cloneCollectionAsCapped with tenant migrations.
   - jstests/core/capped_convertToCapped1.js
+  # These tests run applyOps which is not supported in serverless.
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops2.js
+  - jstests/core/apply_ops_dups.js
+  - jstests/core/apply_ops_index_collation.js
+  - jstests/core/apply_ops_invalid_index_spec.js
+  - jstests/core/apply_ops_missing_field.js
+  - jstests/core/apply_ops_system_dot_views.js
+  - jstests/core/apply_ops_without_ns.js
+  - jstests/core/bypass_doc_validation.js
+  - jstests/core/collation.js
+  - jstests/core/collmod_without_uuid.js
+  - jstests/core/txns/commands_banning_txnnumber_outside_transactions.js
+  - jstests/core/txns/commands_not_allowed_in_txn.js
+  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js
+  - jstests/core/txns/statement_ids_accepted.js
+  - jstests/core/list_collections1.js
+  - jstests/core/list_collections_filter.js
+  - jstests/core/list_collections_no_views.js
+  - jstests/core/views/duplicate_ns.js
+  - jstests/core/views/view_with_invalid_dbname.js
+  - jstests/core/views/views_creation.js
+  - jstests/core/views/invalid_system_views.js
+  - jstests/core/views/views_all_commands.js
+  - jstests/core/rename_stayTemp.js
   # TODO (SERVER-52866): Synchronize getLastError with tenant migrations.
   - jstests/core/bulk_legacy_enforce_gle.js
   # This test contains assertions for the hostname that operations run on.
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml
index 2ae461a6ffb..b19d86a0863 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml
@@ -77,6 +77,32 @@ selector:
   - jstests/core/notablescan_capped.js
   # captrunc command is not blocked during tenant migration.
   - jstests/core/capped6.js
+  # These tests run applyOps which is not supported in serverless.
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops2.js
+  - jstests/core/apply_ops_dups.js
+  - jstests/core/apply_ops_index_collation.js
+  - jstests/core/apply_ops_invalid_index_spec.js
+  - jstests/core/apply_ops_missing_field.js
+  - jstests/core/apply_ops_system_dot_views.js
+  - jstests/core/apply_ops_without_ns.js
+  - jstests/core/bypass_doc_validation.js
+  - jstests/core/collation.js
+  - jstests/core/collmod_without_uuid.js
+  - jstests/core/txns/commands_banning_txnnumber_outside_transactions.js
+  - jstests/core/txns/commands_not_allowed_in_txn.js
+  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js
+  - jstests/core/txns/statement_ids_accepted.js
+  - jstests/core/list_collections1.js
+  - jstests/core/list_collections_filter.js
+  - jstests/core/list_collections_no_views.js
+  - jstests/core/views/duplicate_ns.js
+  - jstests/core/views/view_with_invalid_dbname.js
+  - jstests/core/views/views_creation.js
+  - jstests/core/views/invalid_system_views.js
+  - jstests/core/views/views_all_commands.js
+  - jstests/core/rename_stayTemp.js
 
   #
   # From retryable writes jscore stepdown passthrough
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml
index 8da743c03c1..5a7202a3168 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml
@@ -106,6 +106,32 @@ selector:
   - jstests/core/updatel.js
   - jstests/core/where_system_js.js
   - jstests/core/write_result.js
+  # These tests run applyOps which is not supported in serverless.
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops2.js
+  - jstests/core/apply_ops_dups.js
+  - jstests/core/apply_ops_index_collation.js
+  - jstests/core/apply_ops_invalid_index_spec.js
+  - jstests/core/apply_ops_missing_field.js
+  - jstests/core/apply_ops_system_dot_views.js
+  - jstests/core/apply_ops_without_ns.js
+  - jstests/core/bypass_doc_validation.js
+  - jstests/core/collation.js
+  - jstests/core/collmod_without_uuid.js
+  - jstests/core/txns/commands_banning_txnnumber_outside_transactions.js
+  - jstests/core/txns/commands_not_allowed_in_txn.js
+  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js
+  - jstests/core/txns/statement_ids_accepted.js
+  - jstests/core/list_collections1.js
+  - jstests/core/list_collections_filter.js
+  - jstests/core/list_collections_no_views.js
+  - jstests/core/views/duplicate_ns.js
+  - jstests/core/views/view_with_invalid_dbname.js
+  - jstests/core/views/views_creation.js
+  - jstests/core/views/invalid_system_views.js
+  - jstests/core/views/views_all_commands.js
+  - jstests/core/rename_stayTemp.js
 
   #
   # Excluded from replica_sets_multi_stmt_txn_jscore_passthrough.yml
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml
index 0629f4e3751..f3b45bdd2cb 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml
@@ -77,6 +77,32 @@ selector:
   - jstests/core/notablescan_capped.js
   # captrunc command is not blocked during tenant migration.
   - jstests/core/capped6.js
+  # These tests run applyOps which is not supported in serverless.
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops2.js
+  - jstests/core/apply_ops_dups.js
+  - jstests/core/apply_ops_index_collation.js
+  - jstests/core/apply_ops_invalid_index_spec.js
+  - jstests/core/apply_ops_missing_field.js
+  - jstests/core/apply_ops_system_dot_views.js
+  - jstests/core/apply_ops_without_ns.js
+  - jstests/core/bypass_doc_validation.js
+  - jstests/core/collation.js
+  - jstests/core/collmod_without_uuid.js
+  - jstests/core/txns/commands_banning_txnnumber_outside_transactions.js
+  - jstests/core/txns/commands_not_allowed_in_txn.js
+  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js
+  - jstests/core/txns/statement_ids_accepted.js
+  - jstests/core/list_collections1.js
+  - jstests/core/list_collections_filter.js
+  - jstests/core/list_collections_no_views.js
+  - jstests/core/views/duplicate_ns.js
+  - jstests/core/views/view_with_invalid_dbname.js
+  - jstests/core/views/views_creation.js
+  - jstests/core/views/invalid_system_views.js
+  - jstests/core/views/views_all_commands.js
+  - jstests/core/rename_stayTemp.js
 
   #
   # From retryable writes jscore stepdown passthrough
diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml
index 9e79df5cbe6..43bfc6a938d 100644
--- a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml
@@ -77,6 +77,32 @@ selector:
   - jstests/core/notablescan_capped.js
   # captrunc command is not blocked during tenant migration.
   - jstests/core/capped6.js
+  # These tests run applyOps which is not supported in serverless.
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops1.js
+  - jstests/core/apply_ops2.js
+  - jstests/core/apply_ops_dups.js
+  - jstests/core/apply_ops_index_collation.js
+  - jstests/core/apply_ops_invalid_index_spec.js
+  - jstests/core/apply_ops_missing_field.js
+  - jstests/core/apply_ops_system_dot_views.js
+  - jstests/core/apply_ops_without_ns.js
+  - jstests/core/bypass_doc_validation.js
+  - jstests/core/collation.js
+  - jstests/core/collmod_without_uuid.js
+  - jstests/core/txns/commands_banning_txnnumber_outside_transactions.js
+  - jstests/core/txns/commands_not_allowed_in_txn.js
+  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js
+  - jstests/core/txns/statement_ids_accepted.js
+  - jstests/core/list_collections1.js
+  - jstests/core/list_collections_filter.js
+  - jstests/core/list_collections_no_views.js
+  - jstests/core/views/duplicate_ns.js
+  - jstests/core/views/view_with_invalid_dbname.js
+  - jstests/core/views/views_creation.js
+  - jstests/core/views/invalid_system_views.js
+  - jstests/core/views/views_all_commands.js
+  - jstests/core/rename_stayTemp.js
 
   #
   # From retryable writes jscore stepdown passthrough
diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js
index c512d1567aa..513e9ebde23 100644
--- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js
+++ b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js
@@ -441,6 +441,7 @@ function testRejectBlockedWritesAfterMigrationAborted(testCase, testOpts) {
 
 const isNotWriteCommand = "not a write command";
 const isNotRunOnUserDatabase = "not run on user database";
+const isNotSupportedInServerless = "not supported in serverless cluster";
 const isAuthCommand = "is an auth command";
 const isOnlySupportedOnStandalone = "is only supported on standalone";
 const isOnlySupportedOnShardedCluster = "is only supported on sharded cluster";
@@ -516,85 +517,7 @@ const testCases = {
         }
     },
     appendOplogNote: {skip: isNotRunOnUserDatabase},
-    applyOpsCrudAllowAtomic: {
-        explicitlyCreateCollection: true,
-        command: function(dbName, collName) {
-            return {
-                applyOps: [
-                    {op: "i", ns: dbName + "." + collName, o: {_id: 0}},
-                    {op: "u", ns: dbName + "." + collName, o2: {_id: 0}, o: {$set: {a: 0}}},
-                ],
-                allowAtomic: true,
-            };
-        },
-        assertCommandSucceeded: function(db, dbName, collName) {
-            assert.eq(countDocs(db, collName, {_id: 0, a: 0}), 1);
-        },
-        assertCommandFailed: function(db, dbName, collName) {
-            assert.eq(countDocs(db, collName, {_id: 0}), 0);
-        }
-    },
-    applyOpsCrudNotAllowAtomic: {
-        explicitlyCreateCollection: true,
-        command: function(dbName, collName) {
-            return {
-                applyOps: [
-                    {op: "i", ns: dbName + "." + collName, o: {_id: 0}},
-                    {op: "i", ns: dbName + "." + collName, o: {_id: 1}},
-                    {op: "d", ns: dbName + "." + collName, o: {_id: 1}},
-                ],
-                allowAtomic: false,
-            };
-        },
-        assertCommandSucceeded: function(db, dbName, collName) {
-            assert.eq(countDocs(db, collName, {_id: 0}), 1);
-            assert.eq(countDocs(db, collName, {_id: 1}), 0);
-        },
-        assertCommandFailed: function(db, dbName, collName) {
-            assert.eq(countDocs(db, collName, {_id: 0}), 0);
-            assert.eq(countDocs(db, collName, {_id: 1}), 0);
-        }
-    },
-    applyOpsNonCrudAllowAtomic: {
-        command: function(dbName, collName) {
-            return {
-                applyOps: [
-                    {op: "c", ns: dbName + ".$cmd", o: {create: collName + "1"}},
-                    {op: "c", ns: dbName + ".$cmd", o: {create: collName + "2"}},
-                    {op: "c", ns: dbName + ".$cmd", o: {drop: collName + "2"}},
-                ],
-                allowAtomic: true,
-            };
-        },
-        assertCommandSucceeded: function(db, dbName, collName) {
-            assert(collectionExists(db, collName + "1"));
-            assert(!collectionExists(db, collName + "2"));
-        },
-        assertCommandFailed: function(db, dbName, collName) {
-            assert(!collectionExists(db, collName + "1"));
-            assert(!collectionExists(db, collName + "2"));
-        }
-    },
-    applyOpsNonCrudNotAllowAtomic: {
-        command: function(dbName, collName) {
-            return {
-                applyOps: [
-                    {op: "c", ns: dbName + ".$cmd", o: {create: collName + "1"}},
-                    {op: "c", ns: dbName + ".$cmd", o: {create: collName + "2"}},
-                    {op: "c", ns: dbName + ".$cmd", o: {drop: collName + "2"}},
-                ],
-                allowAtomic: false,
-            };
-        },
-        assertCommandSucceeded: function(db, dbName, collName) {
-            assert(collectionExists(db, collName + "1"));
-            assert(!collectionExists(db, collName + "2"));
-        },
-        assertCommandFailed: function(db, dbName, collName) {
-            assert(!collectionExists(db, collName + "1"));
-            assert(!collectionExists(db, collName + "2"));
-        }
-    },
+    applyOps: {skip: isNotSupportedInServerless},
     authenticate: {skip: isAuthCommand},
     availableQueryOptions: {skip: isNotWriteCommand},
     buildInfo: {skip: isNotWriteCommand},
diff --git a/src/mongo/db/repl/apply_ops.cpp b/src/mongo/db/repl/apply_ops.cpp
index e21fe107a44..840cd23db3b 100644
--- a/src/mongo/db/repl/apply_ops.cpp
+++ b/src/mongo/db/repl/apply_ops.cpp
@@ -51,7 +51,6 @@
 #include "mongo/db/operation_context.h"
 #include "mongo/db/query/collation/collation_spec.h"
 #include "mongo/db/repl/replication_coordinator.h"
-#include "mongo/db/repl/tenant_migration_conflict_info.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/session_catalog_mongod.h"
 #include "mongo/db/transaction_participant.h"
@@ -67,7 +66,7 @@ constexpr StringData ApplyOps::kOplogApplicationModeFieldName;
 
 namespace {
 
-// If enabled, causes loop in _applyOpsWithCommandInfo() to hang after applying current operation.
+// If enabled, causes loop in _applyOps() to hang after applying current operation.
 MONGO_FAIL_POINT_DEFINE(applyOpsPauseBetweenOperations);
 
 /**
@@ -98,12 +97,12 @@ bool _parseAreOpsCrudOnly(const BSONObj& applyOpCmd) {
     return true;
 }
 
-Status _applyOpsWithCommandInfo(OperationContext* opCtx,
-                                const ApplyOpsCommandInfo& info,
-                                repl::OplogApplication::Mode oplogApplicationMode,
-                                BSONObjBuilder* result,
-                                int* numApplied,
-                                BSONArrayBuilder* opsBuilder) {
+Status _applyOps(OperationContext* opCtx,
+                 const ApplyOpsCommandInfo& info,
+                 repl::OplogApplication::Mode oplogApplicationMode,
+                 BSONObjBuilder* result,
+                 int* numApplied,
+                 BSONArrayBuilder* opsBuilder) {
     const auto& ops = info.getOperations();
     // apply
     *numApplied = 0;
@@ -384,11 +383,11 @@ Status applyApplyOpsOplogEntry(OperationContext* opCtx,
                     &resultWeDontCareAbout);
 }
 
-Status _applyOps(OperationContext* opCtx,
-                 const std::string& dbName,
-                 const BSONObj& applyOpCmd,
-                 repl::OplogApplication::Mode oplogApplicationMode,
-                 BSONObjBuilder* result) {
+Status applyOps(OperationContext* opCtx,
+                const std::string& dbName,
+                const BSONObj& applyOpCmd,
+                repl::OplogApplication::Mode oplogApplicationMode,
+                BSONObjBuilder* result) {
     auto info = ApplyOpsCommandInfo::parse(applyOpCmd);
 
     int numApplied = 0;
@@ -426,8 +425,7 @@ Status _applyOps(OperationContext* opCtx,
     }
 
     if (!info.isAtomic()) {
-        return _applyOpsWithCommandInfo(
-            opCtx, info, oplogApplicationMode, result, &numApplied, nullptr);
+        return _applyOps(opCtx, info, oplogApplicationMode, result, &numApplied, nullptr);
     }
 
     // Perform write ops atomically
@@ -445,12 +443,12 @@ Status _applyOps(OperationContext* opCtx,
             {
                 // Suppress replication for atomic operations until end of applyOps.
                 repl::UnreplicatedWritesBlock uwb(opCtx);
-                uassertStatusOK(_applyOpsWithCommandInfo(opCtx,
-                                                         info,
-                                                         oplogApplicationMode,
-                                                         &intermediateResult,
-                                                         &numApplied,
-                                                         opsBuilder.get()));
+                uassertStatusOK(_applyOps(opCtx,
+                                          info,
+                                          oplogApplicationMode,
+                                          &intermediateResult,
+                                          &numApplied,
+                                          opsBuilder.get()));
             }
             // Generate oplog entry for all atomic ops collectively.
             if (opCtx->writesAreReplicated()) {
@@ -485,8 +483,7 @@ Status _applyOps(OperationContext* opCtx,
     } catch (const DBException& ex) {
         if (ex.code() == ErrorCodes::AtomicityFailure) {
             // Retry in non-atomic mode.
-            return _applyOpsWithCommandInfo(
-                opCtx, info, oplogApplicationMode, result, &numApplied, nullptr);
+            return _applyOps(opCtx, info, oplogApplicationMode, result, &numApplied, nullptr);
         }
         BSONArrayBuilder ab;
         ++numApplied;
@@ -503,41 +500,6 @@ Status _applyOps(OperationContext* opCtx,
     return Status::OK();
 }
 
-Status applyOps(OperationContext* opCtx,
-                const std::string& dbName,
-                const BSONObj& applyOpCmd,
-                repl::OplogApplication::Mode oplogApplicationMode,
-                BSONObjBuilder* result) {
-    auto status = _applyOps(opCtx, dbName, applyOpCmd, oplogApplicationMode, result);
-    if (status == ErrorCodes::TenantMigrationConflict) {
-        auto migrationConflictInfo = status.template extraInfo<TenantMigrationConflictInfo>();
-        auto mtab = migrationConflictInfo->getTenantMigrationAccessBlocker();
-        auto migrationStatus =
-            mtab->waitUntilCommittedOrAborted(opCtx, migrationConflictInfo->getOperationType());
-        mtab->recordTenantMigrationError(migrationStatus);
-
-        auto response = result->asTempObj();
-
-        auto numApplied = response.getIntField("applied");
-        BSONArrayBuilder resultsBuilder;
-        BSONObjIterator it(response.getObjectField("results"));
-        while (it.more()) {
-            BSONElement e = it.next();
-            resultsBuilder.append(e.Bool());
-        }
-
-        result->resetToEmpty();
-        result->append("applied", numApplied);
-        result->append("code", migrationStatus.code());
-        result->append("codeName", ErrorCodes::errorString(migrationStatus.code()));
-        result->append("errmsg", migrationStatus.reason());
-        result->append("results", resultsBuilder.arr());
-
-        return migrationStatus;
-    }
-    return status;
-}
-
 // static
 std::vector<OplogEntry> ApplyOps::extractOperations(const OplogEntry& applyOpsOplogEntry) {
     std::vector<OplogEntry> result;