SERVER-46123 Make the dropDatabase command abort in-progress index builds

5 files changed, 124 insertions, 38 deletions

diff --git a/src/mongo/db/catalog/drop_database.cpp b/src/mongo/db/catalog/drop_database.cpp
index f106829cef0..7d6f01eb9dd 100644
--- a/src/mongo/db/catalog/drop_database.cpp
+++ b/src/mongo/db/catalog/drop_database.cpp
@@ -56,9 +56,29 @@ namespace mongo {
 
 MONGO_FAIL_POINT_DEFINE(dropDatabaseHangAfterAllCollectionsDrop);
 MONGO_FAIL_POINT_DEFINE(dropDatabaseHangBeforeInMemoryDrop);
+MONGO_FAIL_POINT_DEFINE(dropDatabaseHangAfterWaitingForIndexBuilds);
 
 namespace {
 
+Status _checkNssAndReplState(OperationContext* opCtx, Database* db, const std::string& dbName) {
+    if (!db) {
+        return Status(ErrorCodes::NamespaceNotFound,
+                      str::stream()
+                          << "Could not drop database " << dbName << " because it does not exist");
+    }
+
+    auto replCoord = repl::ReplicationCoordinator::get(opCtx);
+    bool userInitiatedWritesAndNotPrimary =
+        opCtx->writesAreReplicated() && !replCoord->canAcceptWritesForDatabase(opCtx, dbName);
+
+    if (userInitiatedWritesAndNotPrimary) {
+        return Status(ErrorCodes::NotMaster,
+                      str::stream() << "Not primary while dropping database " << dbName);
+    }
+
+    return Status::OK();
+}
+
 /**
  * Removes database from catalog and writes dropDatabase entry to oplog.
  *
@@ -69,14 +89,17 @@ namespace {
 void _finishDropDatabase(OperationContext* opCtx,
                          const std::string& dbName,
                          Database* db,
-                         std::size_t numCollections) {
+                         std::size_t numCollections,
+                         bool abortIndexBuilds) {
     invariant(opCtx->lockState()->isDbLockedForMode(dbName, MODE_X));
 
     // If DatabaseHolder::dropDb() fails, we should reset the drop-pending state on Database.
     auto dropPendingGuard = makeGuard([db, opCtx] { db->setDropPending(opCtx, false); });
 
-    BackgroundOperation::assertNoBgOpInProgForDb(dbName);
-    IndexBuildsCoordinator::get(opCtx)->assertNoBgOpInProgForDb(dbName);
+    if (!abortIndexBuilds) {
+        BackgroundOperation::assertNoBgOpInProgForDb(dbName);
+        IndexBuildsCoordinator::get(opCtx)->assertNoBgOpInProgForDb(dbName);
+    }
 
     writeConflictRetry(opCtx, "dropDatabase_database", dbName, [&] {
         WriteUnitOfWork wunit(opCtx);
@@ -100,9 +123,7 @@ void _finishDropDatabase(OperationContext* opCtx,
     LOGV2(20336, "dropDatabase {dbName} - finished", "dbName"_attr = dbName);
 }
 
-}  // namespace
-
-Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
+Status _dropDatabase(OperationContext* opCtx, const std::string& dbName, bool abortIndexBuilds) {
     uassert(ErrorCodes::IllegalOperation,
             "Cannot drop a database in read-only mode",
             !storageGlobalParams.readOnly);
@@ -128,20 +149,13 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
     repl::OpTime latestDropPendingOpTime;
 
     {
-        AutoGetDb autoDB(opCtx, dbName, MODE_X);
-        Database* const db = autoDB.getDb();
-        if (!db) {
-            return Status(ErrorCodes::NamespaceNotFound,
-                          str::stream() << "Could not drop database " << dbName
-                                        << " because it does not exist");
-        }
-
-        bool userInitiatedWritesAndNotPrimary =
-            opCtx->writesAreReplicated() && !replCoord->canAcceptWritesForDatabase(opCtx, dbName);
+        boost::optional<AutoGetDb> autoDB;
+        autoDB.emplace(opCtx, dbName, MODE_X);
 
-        if (userInitiatedWritesAndNotPrimary) {
-            return Status(ErrorCodes::NotMaster,
-                          str::stream() << "Not primary while dropping database " << dbName);
+        Database* db = autoDB->getDb();
+        Status status = _checkNssAndReplState(opCtx, db, dbName);
+        if (!status.isOK()) {
+            return status;
         }
 
         if (db->isDropPending(opCtx)) {
@@ -156,6 +170,43 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
         // If Database::dropCollectionEventIfSystem() fails, we should reset the drop-pending state
         // on Database.
         auto dropPendingGuard = makeGuard([&db, opCtx] { db->setDropPending(opCtx, false); });
+        auto indexBuildsCoord = IndexBuildsCoordinator::get(opCtx);
+
+        if (abortIndexBuilds) {
+            // We need to keep aborting all the active index builders for this database until there
+            // are none left when we retrieve the exclusive database lock again.
+            while (indexBuildsCoord->inProgForDb(dbName)) {
+                // Sends the abort signal to all the active index builders for this database.
+                indexBuildsCoord->abortDatabaseIndexBuildsNoWait(dbName, "dropDatabase command");
+
+                // Now that the abort signals were sent out to the active index builders for this
+                // database, we need to release the lock temporarily to allow those index builders
+                // to process the abort signal. Holding a lock here will cause the index builders to
+                // block indefinitely.
+                autoDB = boost::none;
+                indexBuildsCoord->awaitNoBgOpInProgForDb(dbName);
+
+                if (MONGO_unlikely(dropDatabaseHangAfterWaitingForIndexBuilds.shouldFail())) {
+                    LOGV2(4612300,
+                          "dropDatabase - fail point dropDatabaseHangAfterWaitingForIndexBuilds "
+                          "enabled.");
+                    dropDatabaseHangAfterWaitingForIndexBuilds.pauseWhileSet();
+                }
+
+                autoDB.emplace(opCtx, dbName, MODE_X);
+                db = autoDB->getDb();
+
+                // Abandon the snapshot as the index catalog will compare the in-memory state to the
+                // disk state, which may have changed when we released the collection lock
+                // temporarily.
+                opCtx->recoveryUnit()->abandonSnapshot();
+
+                status = _checkNssAndReplState(opCtx, db, dbName);
+                if (!status.isOK()) {
+                    return status;
+                }
+            }
+        }
 
         std::vector<NamespaceString> collectionsToDrop;
         for (auto collIt = db->begin(opCtx); collIt != db->end(opCtx); ++collIt) {
@@ -200,9 +251,11 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
                 invariant(!nss.isReplicated() || nss.coll().startsWith("tmp.mr"));
             }
 
-            BackgroundOperation::assertNoBgOpInProgForNs(nss.ns());
-            IndexBuildsCoordinator::get(opCtx)->assertNoIndexBuildInProgForCollection(
-                CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss)->uuid());
+            if (!abortIndexBuilds) {
+                BackgroundOperation::assertNoBgOpInProgForNs(nss.ns());
+                IndexBuildsCoordinator::get(opCtx)->assertNoIndexBuildInProgForCollection(
+                    CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss)->uuid());
+            }
 
             writeConflictRetry(opCtx, "dropDatabase_collection", nss.ns(), [&] {
                 WriteUnitOfWork wunit(opCtx);
@@ -220,7 +273,7 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
 
         // If there are no collection drops to wait for, we complete the drop database operation.
         if (numCollectionsToDrop == 0U && latestDropPendingOpTime.isNull()) {
-            _finishDropDatabase(opCtx, dbName, db, numCollections);
+            _finishDropDatabase(opCtx, dbName, db, numCollections, abortIndexBuilds);
             return Status::OK();
         }
     }
@@ -344,9 +397,32 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
     // _finishDropDatabase creates its own scope guard to ensure drop-pending is unset.
     dropPendingGuardWhileUnlocked.dismiss();
 
-    _finishDropDatabase(opCtx, dbName, db, numCollections);
+    _finishDropDatabase(opCtx, dbName, db, numCollections, abortIndexBuilds);
 
     return Status::OK();
 }
 
+}  // namespace
+
+Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
+    // Only allow aborting index builds if two-phase index builds are enabled.
+    bool abortIndexBuilds = true;
+    auto indexBuildsCoord = IndexBuildsCoordinator::get(opCtx);
+    if (!indexBuildsCoord->supportsTwoPhaseIndexBuild()) {
+        LOGV2_DEBUG(4612301,
+                    2,
+                    "dropDatabase - not aborting in-progress index builds because two phase index "
+                    "builds are disabled",
+                    "dbName"_attr = dbName);
+        abortIndexBuilds = false;
+    }
+
+    return _dropDatabase(opCtx, dbName, abortIndexBuilds);
+}
+
+Status dropDatabaseForApplyOps(OperationContext* opCtx, const std::string& dbName) {
+    const bool abortIndexBuilds = false;
+    return _dropDatabase(opCtx, dbName, abortIndexBuilds);
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/catalog/drop_database.h b/src/mongo/db/catalog/drop_database.h
index 040b704b9a9..0e3b6e59125 100644
--- a/src/mongo/db/catalog/drop_database.h
+++ b/src/mongo/db/catalog/drop_database.h
@@ -33,7 +33,13 @@ namespace mongo {
 class OperationContext;
 
 /**
- * Drops the database "dbName".
+ * Drops the database "dbName". Aborts in-progress index builds on each collection in the database
+ * if two-phase index builds are enabled.
  */
 Status dropDatabase(OperationContext* opCtx, const std::string& dbName);
+
+/**
+ * Drops the database "dbName". Does not abort in-progress index builds.
+ */
+Status dropDatabaseForApplyOps(OperationContext* opCtx, const std::string& dbName);
 }  // namespace mongo
diff --git a/src/mongo/db/catalog/drop_database_test.cpp b/src/mongo/db/catalog/drop_database_test.cpp
index ea84f806231..da0fc666faa 100644
--- a/src/mongo/db/catalog/drop_database_test.cpp
+++ b/src/mongo/db/catalog/drop_database_test.cpp
@@ -209,7 +209,8 @@ void _removeDatabaseFromCatalog(OperationContext* opCtx, StringData dbName) {
 
 TEST_F(DropDatabaseTest, DropDatabaseReturnsNamespaceNotFoundIfDatabaseDoesNotExist) {
     ASSERT_FALSE(AutoGetDb(_opCtx.get(), _nss.db(), MODE_X).getDb());
-    ASSERT_EQUALS(ErrorCodes::NamespaceNotFound, dropDatabase(_opCtx.get(), _nss.db().toString()));
+    ASSERT_EQUALS(ErrorCodes::NamespaceNotFound,
+                  dropDatabaseForApplyOps(_opCtx.get(), _nss.db().toString()));
 }
 
 TEST_F(DropDatabaseTest, DropDatabaseReturnsNotMasterIfNotPrimary) {
@@ -217,7 +218,8 @@ TEST_F(DropDatabaseTest, DropDatabaseReturnsNotMasterIfNotPrimary) {
     ASSERT_OK(_replCoord->setFollowerMode(repl::MemberState::RS_SECONDARY));
     ASSERT_TRUE(_opCtx->writesAreReplicated());
     ASSERT_FALSE(_replCoord->canAcceptWritesForDatabase(_opCtx.get(), _nss.db()));
-    ASSERT_EQUALS(ErrorCodes::NotMaster, dropDatabase(_opCtx.get(), _nss.db().toString()));
+    ASSERT_EQUALS(ErrorCodes::NotMaster,
+                  dropDatabaseForApplyOps(_opCtx.get(), _nss.db().toString()));
 }
 
 /**
@@ -238,7 +240,7 @@ void _testDropDatabase(OperationContext* opCtx,
     ASSERT_TRUE(db);
     opObserver->db = db;
 
-    ASSERT_OK(dropDatabase(opCtx, nss.db().toString()));
+    ASSERT_OK(dropDatabaseForApplyOps(opCtx, nss.db().toString()));
     ASSERT_FALSE(AutoGetDb(opCtx, nss.db(), MODE_X).getDb());
     opObserver->db = nullptr;
 
@@ -293,7 +295,8 @@ TEST_F(DropDatabaseTest, DropDatabasePassedThroughAwaitReplicationErrorForDropPe
     auto dpns = _nss.makeDropPendingNamespace(dropOpTime);
     _createCollection(_opCtx.get(), dpns);
 
-    ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, dropDatabase(_opCtx.get(), _nss.db().toString()));
+    ASSERT_EQUALS(ErrorCodes::WriteConcernFailed,
+                  dropDatabaseForApplyOps(_opCtx.get(), _nss.db().toString()));
 }
 
 TEST_F(DropDatabaseTest, DropDatabaseSkipsSystemProfileCollectionWhenDroppingCollections) {
@@ -312,7 +315,7 @@ TEST_F(DropDatabaseTest, DropDatabaseResetsDropPendingStateOnException) {
     auto db = autoDb.getDb();
     ASSERT_TRUE(db);
 
-    ASSERT_THROWS_CODE_AND_WHAT(dropDatabase(_opCtx.get(), _nss.db().toString()),
+    ASSERT_THROWS_CODE_AND_WHAT(dropDatabaseForApplyOps(_opCtx.get(), _nss.db().toString()),
                                 AssertionException,
                                 ErrorCodes::OperationFailed,
                                 "onDropCollection() failed");
@@ -327,7 +330,8 @@ void _testDropDatabaseResetsDropPendingStateIfAwaitReplicationFails(OperationCon
 
     ASSERT_TRUE(AutoGetDb(opCtx, nss.db(), MODE_X).getDb());
 
-    ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, dropDatabase(opCtx, nss.db().toString()));
+    ASSERT_EQUALS(ErrorCodes::WriteConcernFailed,
+                  dropDatabaseForApplyOps(opCtx, nss.db().toString()));
 
     AutoGetDb autoDb(opCtx, nss.db(), MODE_X);
     auto db = autoDb.getDb();
@@ -410,7 +414,7 @@ TEST_F(DropDatabaseTest,
 
     {
         Lock::DBLock lk(_opCtx.get(), _nss.db(), MODE_X);
-        ASSERT_OK(dropDatabase(_opCtx.get(), _nss.db().toString()));
+        ASSERT_OK(dropDatabaseForApplyOps(_opCtx.get(), _nss.db().toString()));
     }
 
     ASSERT_TRUE(isAwaitReplicationCalled);
@@ -429,7 +433,7 @@ TEST_F(DropDatabaseTest,
 
     ASSERT_TRUE(AutoGetDb(_opCtx.get(), _nss.db(), MODE_X).getDb());
 
-    auto status = dropDatabase(_opCtx.get(), _nss.db().toString());
+    auto status = dropDatabaseForApplyOps(_opCtx.get(), _nss.db().toString());
     ASSERT_EQUALS(ErrorCodes::NamespaceNotFound, status);
     ASSERT_EQUALS(status.reason(),
                   std::string(str::stream()
@@ -454,7 +458,7 @@ TEST_F(DropDatabaseTest,
 
     ASSERT_TRUE(AutoGetDb(_opCtx.get(), _nss.db(), MODE_X).getDb());
 
-    auto status = dropDatabase(_opCtx.get(), _nss.db().toString());
+    auto status = dropDatabaseForApplyOps(_opCtx.get(), _nss.db().toString());
     ASSERT_EQUALS(ErrorCodes::PrimarySteppedDown, status);
     ASSERT_EQUALS(status.reason(),
                   std::string(str::stream() << "Could not drop database " << _nss.db()
@@ -471,7 +475,7 @@ TEST_F(DropDatabaseTest,
 TEST_F(DropDatabaseTest, DropDatabaseFailsToDropAdmin) {
     NamespaceString adminNSS(NamespaceString::kAdminDb, "foo");
     _createCollection(_opCtx.get(), adminNSS);
-    ASSERT_THROWS_CODE_AND_WHAT(dropDatabase(_opCtx.get(), adminNSS.db().toString()),
+    ASSERT_THROWS_CODE_AND_WHAT(dropDatabaseForApplyOps(_opCtx.get(), adminNSS.db().toString()),
                                 AssertionException,
                                 ErrorCodes::IllegalOperation,
                                 "Dropping the 'admin' database is prohibited.");
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index 33ba5e3d75d..ef6c546f2c3 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -795,7 +795,7 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
     {"dbCheck", {dbCheckOplogCommand, {}}},
     {"dropDatabase",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
-          return dropDatabase(opCtx, entry.getNss().db().toString());
+          return dropDatabaseForApplyOps(opCtx, entry.getNss().db().toString());
       },
       {ErrorCodes::NamespaceNotFound}}},
     {"drop",
diff --git a/src/mongo/dbtests/storage_timestamp_tests.cpp b/src/mongo/dbtests/storage_timestamp_tests.cpp
index 6e3090feeac..b9b2e0624ee 100644
--- a/src/mongo/dbtests/storage_timestamp_tests.cpp
+++ b/src/mongo/dbtests/storage_timestamp_tests.cpp
@@ -1803,11 +1803,11 @@ public:
 
         const Timestamp dropTime = _clock->reserveTicks(1).asTimestamp();
         if (SimulatePrimary) {
-            ASSERT_OK(dropDatabase(_opCtx, nss.db().toString()));
+            ASSERT_OK(dropDatabaseForApplyOps(_opCtx, nss.db().toString()));
         } else {
             repl::UnreplicatedWritesBlock uwb(_opCtx);
             TimestampBlock ts(_opCtx, dropTime);
-            ASSERT_OK(dropDatabase(_opCtx, nss.db().toString()));
+            ASSERT_OK(dropDatabaseForApplyOps(_opCtx, nss.db().toString()));
         }
 
         // Assert that the idents do not exist.