SERVER-29277 dropDatabase() waits for collection drops to be replicated before dropping database

2 files changed, 155 insertions, 9 deletions

diff --git a/src/mongo/db/catalog/drop_database.cpp b/src/mongo/db/catalog/drop_database.cpp
index 90e9adb34f7..8ccc1c8f944 100644
--- a/src/mongo/db/catalog/drop_database.cpp
+++ b/src/mongo/db/catalog/drop_database.cpp
@@ -41,11 +41,42 @@
 #include "mongo/db/op_observer.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/service_context.h"
+#include "mongo/db/write_concern_options.h"
 #include "mongo/util/log.h"
 #include "mongo/util/scopeguard.h"
 
 namespace mongo {
 
+namespace {
+
+// This is used to wait for the collection drops to replicate to a majority of the replica set.
+// Note: Even though we're setting UNSET here, kMajority implies JOURNAL if journaling is supported
+// by mongod and writeConcernMajorityJournalDefault is set to true in the ReplSetConfig.
+const WriteConcernOptions kDropDatabaseWriteConcern(WriteConcernOptions::kMajority,
+                                                    WriteConcernOptions::SyncMode::UNSET,
+                                                    Minutes(10));
+
+/**
+ * Removes database from catalog and writes dropDatabase entry to oplog.
+ */
+Status _finishDropDatabase(OperationContext* opCtx, const std::string& dbName, Database* db) {
+    // If Database::dropDatabase() fails, we should reset the drop-pending state on Database.
+    auto dropPendingGuard = MakeGuard([db, opCtx] { db->setDropPending(opCtx, false); });
+
+    Database::dropDatabase(opCtx, db);
+    dropPendingGuard.Dismiss();
+
+    log() << "dropDatabase " << dbName << " - finished";
+
+    WriteUnitOfWork wunit(opCtx);
+    getGlobalServiceContext()->getOpObserver()->onDropDatabase(opCtx, dbName);
+    wunit.commit();
+
+    return Status::OK();
+}
+
+}  // namespace
+
 Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
     uassert(ErrorCodes::IllegalOperation,
             "Cannot drop a database in read-only mode",
@@ -57,6 +88,9 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
         CurOp::get(opCtx)->setNS_inlock(dbName);
     }
 
+    auto replCoord = repl::ReplicationCoordinator::get(opCtx);
+    std::size_t numCollectionsToDrop = 0;
+
     MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
         Lock::GlobalWrite lk(opCtx);
         AutoGetDb autoDB(opCtx, dbName, MODE_X);
@@ -67,7 +101,6 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
                                         << " because it does not exist");
         }
 
-        auto replCoord = repl::ReplicationCoordinator::get(opCtx);
         bool userInitiatedWritesAndNotPrimary =
             opCtx->writesAreReplicated() && !replCoord->canAcceptWritesForDatabase(opCtx, dbName);
 
@@ -79,7 +112,8 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
         log() << "dropDatabase " << dbName << " - starting";
         db->setDropPending(opCtx, true);
 
-        // If Database::dropDatabase() fails, we should reset the drop-pending state on Database.
+        // If Database::dropCollectionEventIfSystem() fails, we should reset the drop-pending state
+        // on Database.
         auto dropPendingGuard = MakeGuard([&db, opCtx] { db->setDropPending(opCtx, false); });
 
         for (auto collection : *db) {
@@ -91,20 +125,57 @@ Status dropDatabase(OperationContext* opCtx, const std::string& dbName) {
             WriteUnitOfWork wunit(opCtx);
             fassertStatusOK(40476, db->dropCollectionEvenIfSystem(opCtx, nss));
             wunit.commit();
+            numCollectionsToDrop++;
         }
-        Database::dropDatabase(opCtx, db);
         dropPendingGuard.Dismiss();
-        log() << "dropDatabase " << dbName << " - finished";
 
-        WriteUnitOfWork wunit(opCtx);
+        // If there are no collection drops to wait for, we complete the drop database operation.
+        if (numCollectionsToDrop == 0U) {
+            return _finishDropDatabase(opCtx, dbName, db);
+        }
+    }
+    MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "dropDatabase_collection", dbName);
+
+    // If waitForWriteConcern() returns an error or throws an exception, we should reset the
+    // drop-pending state on Database.
+    auto dropPendingGuardWhileAwaitingReplication = MakeGuard([dbName, opCtx] {
+        Lock::GlobalWrite lk(opCtx);
+        AutoGetDb autoDB(opCtx, dbName, MODE_X);
+        if (auto db = autoDB.getDb()) {
+            db->setDropPending(opCtx, false);
+        }
+    });
+
+    auto status =
+        replCoord->awaitReplicationOfLastOpForClient(opCtx, kDropDatabaseWriteConcern).status;
+    if (!status.isOK()) {
+        return Status(status.code(),
+                      str::stream() << "dropDatabase " << dbName << " failed waiting for "
+                                    << numCollectionsToDrop
+                                    << " collection drops to replicate: "
+                                    << status.reason());
+    }
 
-        getGlobalServiceContext()->getOpObserver()->onDropDatabase(opCtx, dbName);
+    log() << "dropDatabase " << dbName << " - successfully dropped " << numCollectionsToDrop
+          << " collections. dropping database";
+    dropPendingGuardWhileAwaitingReplication.Dismiss();
 
-        wunit.commit();
+    MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
+        Lock::GlobalWrite lk(opCtx);
+        AutoGetDb autoDB(opCtx, dbName, MODE_X);
+        if (auto db = autoDB.getDb()) {
+            return _finishDropDatabase(opCtx, dbName, db);
+        }
+
+        return Status(ErrorCodes::NamespaceNotFound,
+                      str::stream() << "Could not drop database " << dbName
+                                    << " because it does not exist after dropping "
+                                    << numCollectionsToDrop
+                                    << " collection(s).");
     }
-    MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "dropDatabase", dbName);
+    MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "dropDatabase_database", dbName);
 
-    return Status::OK();
+    MONGO_UNREACHABLE;
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/catalog/drop_database_test.cpp b/src/mongo/db/catalog/drop_database_test.cpp
index 43a2e4c1dfb..ab83ac01521 100644
--- a/src/mongo/db/catalog/drop_database_test.cpp
+++ b/src/mongo/db/catalog/drop_database_test.cpp
@@ -50,6 +50,7 @@
 #include "mongo/stdx/memory.h"
 #include "mongo/unittest/unittest.h"
 #include "mongo/util/assert_util.h"
+#include "mongo/util/mongoutils/str.h"
 
 namespace {
 
@@ -179,6 +180,17 @@ void _createCollection(OperationContext* opCtx, const NamespaceString& nss) {
     ASSERT_TRUE(AutoGetCollectionForRead(opCtx, nss).getCollection());
 }
 
+/**
+ * Removes database from catalog, bypassing dropDatabase().
+ */
+void _removeDatabaseFromCatalog(OperationContext* opCtx, StringData dbName) {
+    Lock::GlobalWrite lk(opCtx);
+    AutoGetDb autoDB(opCtx, dbName, MODE_X);
+    auto db = autoDB.getDb();
+    ASSERT_TRUE(db);
+    Database::dropDatabase(opCtx, db);
+}
+
 TEST_F(DropDatabaseTest, DropDatabaseReturnsNamespaceNotFoundIfDatabaseDoesNotExist) {
     ASSERT_FALSE(AutoGetDb(_opCtx.get(), _nss.db(), MODE_X).getDb());
     ASSERT_EQUALS(ErrorCodes::NamespaceNotFound, dropDatabase(_opCtx.get(), _nss.db().toString()));
@@ -274,4 +286,67 @@ TEST_F(DropDatabaseTest, DropDatabaseResetsDropPendingStateOnException) {
     ASSERT_FALSE(db->isDropPending(_opCtx.get()));
 }
 
+void _testDropDatabaseResetsDropPendingStateIfAwaitReplicationFails(OperationContext* opCtx,
+                                                                    const NamespaceString& nss,
+                                                                    bool expectDbPresent) {
+    _createCollection(opCtx, nss);
+
+    ASSERT_TRUE(AutoGetDb(opCtx, nss.db(), MODE_X).getDb());
+
+    ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, dropDatabase(opCtx, nss.db().toString()));
+
+    AutoGetDb autoDb(opCtx, nss.db(), MODE_X);
+    auto db = autoDb.getDb();
+    if (expectDbPresent) {
+        ASSERT_TRUE(db);
+        ASSERT_FALSE(db->isDropPending(opCtx));
+    } else {
+        ASSERT_FALSE(db);
+    }
+}
+
+TEST_F(DropDatabaseTest,
+       DropDatabaseResetsDropPendingStateIfAwaitReplicationFailsAndDatabaseIsPresent) {
+    // Update ReplicationCoordinatorMock so that awaitReplicationOfLastOpForClient() fails.
+    _replCoord->setAwaitReplicationReturnValueFunction([] {
+        return repl::ReplicationCoordinator::StatusAndDuration(
+            Status(ErrorCodes::WriteConcernFailed, ""), Milliseconds(0));
+    });
+
+    _testDropDatabaseResetsDropPendingStateIfAwaitReplicationFails(_opCtx.get(), _nss, true);
+}
+
+TEST_F(DropDatabaseTest,
+       DropDatabaseResetsDropPendingStateIfAwaitReplicationFailsAndDatabaseIsMissing) {
+    // Update ReplicationCoordinatorMock so that awaitReplicationOfLastOpForClient() fails.
+    _replCoord->setAwaitReplicationReturnValueFunction([this] {
+        _removeDatabaseFromCatalog(_opCtx.get(), _nss.db());
+        return repl::ReplicationCoordinator::StatusAndDuration(
+            Status(ErrorCodes::WriteConcernFailed, ""), Milliseconds(0));
+    });
+
+    _testDropDatabaseResetsDropPendingStateIfAwaitReplicationFails(_opCtx.get(), _nss, false);
+}
+
+TEST_F(DropDatabaseTest,
+       DropDatabaseReturnsNamespaceNotFoundIfDatabaseIsRemovedAfterCollectionsDropsAreReplicated) {
+    // Update ReplicationCoordinatorMock so that awaitReplicationOfLastOpForClient() fails.
+    _replCoord->setAwaitReplicationReturnValueFunction([this] {
+        _removeDatabaseFromCatalog(_opCtx.get(), _nss.db());
+        return repl::ReplicationCoordinator::StatusAndDuration(Status::OK(), Milliseconds(0));
+    });
+
+    _createCollection(_opCtx.get(), _nss);
+
+    ASSERT_TRUE(AutoGetDb(_opCtx.get(), _nss.db(), MODE_X).getDb());
+
+    auto status = dropDatabase(_opCtx.get(), _nss.db().toString());
+    ASSERT_EQUALS(ErrorCodes::NamespaceNotFound, status);
+    ASSERT_EQUALS(status.reason(),
+                  str::stream() << "Could not drop database " << _nss.db()
+                                << " because it does not exist after dropping 1 collection(s).");
+
+    ASSERT_FALSE(AutoGetDb(_opCtx.get(), _nss.db(), MODE_X).getDb());
+}
+
 }  // namespace