SERVER-39714 Drop unfinished indexes on modified collections during repair when run on a replica set node

4 files changed, 57 insertions, 17 deletions

diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp
index 07fed07a646..9a2b52d9967 100644
--- a/src/mongo/db/repair_database.cpp
+++ b/src/mongo/db/repair_database.cpp
@@ -32,6 +32,7 @@
 #include "mongo/platform/basic.h"
 
 #include <algorithm>
+#include <fmt/format.h>
 
 #include "mongo/db/repair_database.h"
 
@@ -56,14 +57,18 @@
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/query/query_knobs_gen.h"
 #include "mongo/db/rebuild_indexes.h"
+#include "mongo/db/repl_set_member_in_standalone_mode.h"
 #include "mongo/db/storage/durable_catalog.h"
 #include "mongo/db/storage/storage_engine.h"
+#include "mongo/db/storage/storage_repair_observer.h"
 #include "mongo/logv2/log.h"
 #include "mongo/util/log.h"
 #include "mongo/util/scopeguard.h"
 
 namespace mongo {
 
+using namespace fmt::literals;
+
 Status rebuildIndexesForNamespace(OperationContext* opCtx,
                                   const NamespaceString& nss,
                                   StorageEngine* engine) {
@@ -83,6 +88,31 @@ Status rebuildIndexesForNamespace(OperationContext* opCtx,
 }
 
 namespace {
+Status dropUnfinishedIndexes(OperationContext* opCtx, Collection* collection) {
+    std::vector<std::string> indexNames;
+    auto durableCatalog = DurableCatalog::get(opCtx);
+    durableCatalog->getAllIndexes(opCtx, collection->getCatalogId(), &indexNames);
+    for (const auto& indexName : indexNames) {
+        if (!durableCatalog->isIndexReady(opCtx, collection->getCatalogId(), indexName)) {
+            LOGV2(3871400,
+                  "Dropping unfinished index '{name}' after collection was modified by "
+                  "repair",
+                  "name"_attr = indexName);
+            WriteUnitOfWork wuow(opCtx);
+            auto status = durableCatalog->removeIndex(opCtx, collection->getCatalogId(), indexName);
+            if (!status.isOK()) {
+                return status;
+            }
+            wuow.commit();
+            StorageRepairObserver::get(opCtx->getServiceContext())
+                ->invalidatingModification(str::stream()
+                                           << "Dropped unfinished index '" << indexName << "' on "
+                                           << collection->ns());
+        }
+    }
+    return Status::OK();
+}
+
 Status repairCollections(OperationContext* opCtx,
                          StorageEngine* engine,
                          const std::string& dbName) {
@@ -96,9 +126,27 @@ Status repairCollections(OperationContext* opCtx,
         auto collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss);
         Status status = engine->repairRecordStore(opCtx, collection->getCatalogId(), nss);
 
+        // Need to lookup from catalog again because the old collection object was invalidated by
+        // repairRecordStore.
+        collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss);
+
         // If data was modified during repairRecordStore, we know to rebuild indexes without needing
         // to run an expensive collection validation.
         if (status.code() == ErrorCodes::DataModifiedByRepair) {
+            invariant(StorageRepairObserver::get(opCtx->getServiceContext())->isDataInvalidated(),
+                      "Collection '{}' ({})"_format(collection->ns().toString(),
+                                                    collection->uuid().toString()));
+
+            // If we are a replica set member in standalone mode and we have unfinished indexes,
+            // drop them before rebuilding any completed indexes. Since we have already made
+            // invalidating modifications to our data, it is safe to just drop the indexes entirely
+            // to avoid the risk of the index rebuild failing.
+            if (getReplSetMemberInStandaloneMode(opCtx->getServiceContext())) {
+                if (auto status = dropUnfinishedIndexes(opCtx, collection); !status.isOK()) {
+                    return status;
+                }
+            }
+
             Status status = rebuildIndexesForNamespace(opCtx, nss, engine);
             if (!status.isOK()) {
                 return status;
@@ -109,10 +157,7 @@ Status repairCollections(OperationContext* opCtx,
         }
 
         // Run collection validation to avoid unecessarily rebuilding indexes on valid collections
-        // with consistent indexes. Initialize the collection prior to validation. Need to lookup
-        // from catalog again because the old collection object was invalidated by
-        // repairRecordStore.
-        collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss);
+        // with consistent indexes. Initialize the collection prior to validation.
         collection->init(opCtx);
 
         ValidateResults validateResults;
diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp
index 5aab053045a..bbc6cd8f75d 100644
--- a/src/mongo/db/repair_database_and_check_version.cpp
+++ b/src/mongo/db/repair_database_and_check_version.cpp
@@ -408,9 +408,11 @@ bool repairDatabasesAndCheckVersion(OperationContext* opCtx) {
         for (const auto& dbName : dbNames) {
             LOGV2_DEBUG(21007, 1, "    Repairing database: {dbName}", "dbName"_attr = dbName);
             fassertNoTrace(18506, repairDatabase(opCtx, storageEngine, dbName));
-        }
 
-        setReplSetMemberInStandaloneMode(opCtx);
+            if (dbName == NamespaceString::kLocalDb) {
+                setReplSetMemberInStandaloneMode(opCtx);
+            }
+        }
 
         // All collections must have UUIDs before restoring the FCV document to a version that
         // requires UUIDs.
diff --git a/src/mongo/db/repl_set_member_in_standalone_mode.cpp b/src/mongo/db/repl_set_member_in_standalone_mode.cpp
index 4ea8bc6bc1e..9859ab9333b 100644
--- a/src/mongo/db/repl_set_member_in_standalone_mode.cpp
+++ b/src/mongo/db/repl_set_member_in_standalone_mode.cpp
@@ -35,24 +35,17 @@
 namespace mongo {
 namespace {
 
-// The boost::optional<bool> decoration will be initialized to boost::none.
-const auto& replSetMemberInStandaloneMode =
-    ServiceContext::declareDecoration<boost::optional<bool>>();
+const auto& replSetMemberInStandaloneMode = ServiceContext::declareDecoration<bool>();
 
 }  // namespace
 
 bool getReplSetMemberInStandaloneMode(ServiceContext* serviceCtx) {
-    auto& replSetMemberInStandaloneModeBool = replSetMemberInStandaloneMode(serviceCtx);
-    invariant(replSetMemberInStandaloneModeBool);
-
-    return replSetMemberInStandaloneModeBool.get();
+    return replSetMemberInStandaloneMode(serviceCtx);
 }
 
 void setReplSetMemberInStandaloneMode(ServiceContext* serviceCtx,
                                       bool isReplSetMemberInStandaloneMode) {
     auto& replSetMemberInStandaloneModeBool = replSetMemberInStandaloneMode(serviceCtx);
-    invariant(!replSetMemberInStandaloneModeBool);
-
     replSetMemberInStandaloneModeBool = isReplSetMemberInStandaloneMode;
 }
 
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index bf805eef4f0..8ebc5218ffb 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -361,8 +361,8 @@ public:
      * Recover as much data as possible from a potentially corrupt RecordStore.
      * This only recovers the record data, not indexes or anything else.
      *
-     * Generally, this method should not be called directly except by the repairDatabase()
-     * free function.
+     * The Collection object for on this namespace will be destructed and invalidated. A new
+     * Collection object will be created and it should be retrieved from the CollectionCatalog.
      */
     virtual Status repairRecordStore(OperationContext* opCtx,
                                      RecordId catalogId,