SERVER-37637 Standalone nodes should not restart in-progress two-phase index builds on startup

If a replica set node is started in standalone mode and there are
incomplete two-phase index builds, instead of restarting them, we mark
the indexes as 'frozen'. This signifies that the indexes are unusable,
and will not be rebuilt. The only option available to users is to drop
the index. This can corrupt the replica set, but modifying replicated
data in standalone mode already poses this danger.

10 files changed, 145 insertions, 53 deletions

diff --git a/src/mongo/db/catalog/drop_indexes.cpp b/src/mongo/db/catalog/drop_indexes.cpp
index 6a14fa7505f..8f29522b82f 100644
--- a/src/mongo/db/catalog/drop_indexes.cpp
+++ b/src/mongo/db/catalog/drop_indexes.cpp
@@ -43,6 +43,7 @@
 #include "mongo/db/index_builds_coordinator.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/repl_set_member_in_standalone_mode.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/views/view_catalog.h"
 #include "mongo/util/log.h"
@@ -58,22 +59,33 @@ namespace {
 constexpr auto kIndexFieldName = "index"_sd;
 
 /**
- * Drops single index by name.
+ * Drops single index given a descriptor.
  */
-Status dropIndexByName(OperationContext* opCtx,
-                       Collection* collection,
-                       IndexCatalog* indexCatalog,
-                       const std::string& indexToDelete) {
-    auto desc = indexCatalog->findIndexByName(opCtx, indexToDelete);
-    if (!desc) {
-        return Status(ErrorCodes::IndexNotFound,
-                      str::stream() << "index not found with name [" << indexToDelete << "]");
-    }
-
+Status dropIndexByDescriptor(OperationContext* opCtx,
+                             Collection* collection,
+                             IndexCatalog* indexCatalog,
+                             const IndexDescriptor* desc) {
     if (desc->isIdIndex()) {
         return Status(ErrorCodes::InvalidOptions, "cannot drop _id index");
     }
 
+    // Support dropping unfinished indexes, but only if the index is 'frozen'. These indexes only
+    // exist in standalone mode.
+    auto entry = indexCatalog->getEntry(desc);
+    if (entry->isFrozen()) {
+        invariant(!entry->isReady(opCtx));
+        invariant(getReplSetMemberInStandaloneMode(opCtx->getServiceContext()));
+        // Return here. No need to fall through to op observer on standalone.
+        return indexCatalog->dropUnfinishedIndex(opCtx, desc);
+    }
+
+    // Do not allow dropping unfinished indexes that are not frozen.
+    if (!entry->isReady(opCtx)) {
+        return Status(ErrorCodes::IndexNotFound,
+                      str::stream()
+                          << "can't drop unfinished index with name: " << desc->indexName());
+    }
+
     auto s = indexCatalog->dropIndex(opCtx, desc);
     if (!s.isOK()) {
         return s;
@@ -111,13 +123,20 @@ Status wrappedRun(OperationContext* opCtx,
             return Status::OK();
         }
 
-        return dropIndexByName(opCtx, collection, indexCatalog, indexToDelete);
+        bool includeUnfinished = true;
+        auto desc = indexCatalog->findIndexByName(opCtx, indexToDelete, includeUnfinished);
+        if (!desc) {
+            return Status(ErrorCodes::IndexNotFound,
+                          str::stream() << "index not found with name [" << indexToDelete << "]");
+        }
+        return dropIndexByDescriptor(opCtx, collection, indexCatalog, desc);
     }
 
     if (indexElem.type() == Object) {
+        const bool includeUnfinished = true;
         std::vector<const IndexDescriptor*> indexes;
         collection->getIndexCatalog()->findIndexesByKeyPattern(
-            opCtx, indexElem.embeddedObject(), false, &indexes);
+            opCtx, indexElem.embeddedObject(), includeUnfinished, &indexes);
         if (indexes.empty()) {
             return Status(ErrorCodes::IndexNotFound,
                           str::stream()
@@ -146,15 +165,7 @@ Status wrappedRun(OperationContext* opCtx,
                           "name of '*', or downgrade to 3.4 to drop only this index.");
         }
 
-        Status s = indexCatalog->dropIndex(opCtx, desc);
-        if (!s.isOK()) {
-            return s;
-        }
-
-        opCtx->getServiceContext()->getOpObserver()->onDropIndex(
-            opCtx, collection->ns(), collection->uuid(), desc->indexName(), desc->infoObj());
-
-        return Status::OK();
+        return dropIndexByDescriptor(opCtx, collection, indexCatalog, desc);
     }
 
     // The 'index' field contains a list of names of indexes to drop.
@@ -170,7 +181,14 @@ Status wrappedRun(OperationContext* opCtx,
             }
 
             auto indexToDelete = indexNameElem.String();
-            auto status = dropIndexByName(opCtx, collection, indexCatalog, indexToDelete);
+            bool includeUnfinished = true;
+            auto desc = indexCatalog->findIndexByName(opCtx, indexToDelete, includeUnfinished);
+            if (!desc) {
+                return Status(ErrorCodes::IndexNotFound,
+                              str::stream()
+                                  << "index not found with name [" << indexToDelete << "]");
+            }
+            auto status = dropIndexByDescriptor(opCtx, collection, indexCatalog, desc);
             if (!status.isOK()) {
                 return status.withContext(
                     str::stream() << "dropIndexes " << collection->ns() << " ("
diff --git a/src/mongo/db/catalog/index_build_block.cpp b/src/mongo/db/catalog/index_build_block.cpp
index ba0121f0f33..1efc401f13c 100644
--- a/src/mongo/db/catalog/index_build_block.cpp
+++ b/src/mongo/db/catalog/index_build_block.cpp
@@ -104,10 +104,8 @@ Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection) {
     if (!status.isOK())
         return status;
 
-    const bool initFromDisk = false;
-    const bool isReadyIndex = false;
     _indexCatalogEntry =
-        _indexCatalog->createIndexEntry(opCtx, std::move(descriptor), initFromDisk, isReadyIndex);
+        _indexCatalog->createIndexEntry(opCtx, std::move(descriptor), CreateIndexEntryFlags::kNone);
 
     // Only track skipped records with two-phase index builds, which is indicated by a present build
     // UUID.
diff --git a/src/mongo/db/catalog/index_catalog.h b/src/mongo/db/catalog/index_catalog.h
index b3cda2d6466..9194874f386 100644
--- a/src/mongo/db/catalog/index_catalog.h
+++ b/src/mongo/db/catalog/index_catalog.h
@@ -77,6 +77,34 @@ enum class IndexBuildMethod {
     kForeground,
 };
 
+enum class CreateIndexEntryFlags : int {
+    kNone = 0x0,
+    /**
+     * kInitFromDisk avoids registering a change to undo this operation when set to true. You
+     * must set this flag if calling this function outside of a WriteUnitOfWork.
+     */
+    kInitFromDisk = 0x1,
+    /**
+     * kIsReady controls whether the index will be directly available for query usage without
+     * needing to complete the IndexBuildBlock process.
+     */
+    kIsReady = 0x2,
+    /**
+     * kFrozen indicates that the index is not usable and that it is not currently being
+     * built. This is used when starting a node in standalone mode and a two-phase index build
+     * is incomplete. kIsReady must not also be set.
+     */
+    kFrozen = 0x4
+};
+
+inline bool operator&(CreateIndexEntryFlags lhs, CreateIndexEntryFlags rhs) {
+    return (static_cast<int>(lhs) & static_cast<int>(rhs)) != 0;
+}
+
+inline CreateIndexEntryFlags operator|(CreateIndexEntryFlags lhs, CreateIndexEntryFlags rhs) {
+    return CreateIndexEntryFlags(static_cast<int>(lhs) | static_cast<int>(rhs));
+}
+
 /**
  * The IndexCatalog is owned by the Collection and is responsible for the lookup and lifetimes of
  * the indexes in a collection. Every collection has exactly one instance of this class.
@@ -284,16 +312,11 @@ public:
     /*
      * Creates an index entry with the provided descriptor on the catalog's collection.
      *
-     * 'initFromDisk' avoids registering a change to undo this operation when set to true. You must
-     * set this flag if calling this function outside of a WriteUnitOfWork.
-     *
-     * 'isReadyIndex' controls whether the index will be directly available for query usage without
-     * needing to complete the IndexBuildBlock process.
      */
+
     virtual IndexCatalogEntry* createIndexEntry(OperationContext* opCtx,
                                                 std::unique_ptr<IndexDescriptor> descriptor,
-                                                bool initFromDisk,
-                                                bool isReadyIndex) = 0;
+                                                CreateIndexEntryFlags flags) = 0;
 
     /**
      * Call this only on an empty collection from inside a WriteUnitOfWork. Index creation on an
diff --git a/src/mongo/db/catalog/index_catalog_entry.h b/src/mongo/db/catalog/index_catalog_entry.h
index 67ddd0d31b6..f65194fe62b 100644
--- a/src/mongo/db/catalog/index_catalog_entry.h
+++ b/src/mongo/db/catalog/index_catalog_entry.h
@@ -133,6 +133,12 @@ public:
     // if this ready is ready for queries
     virtual bool isReady(OperationContext* const opCtx) const = 0;
 
+    /**
+     * Returns true if this index is not ready, and it is not currently in the process of being
+     * built either.
+     */
+    virtual bool isFrozen() const = 0;
+
     virtual KVPrefix getPrefix() const = 0;
 
     /**
diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.cpp b/src/mongo/db/catalog/index_catalog_entry_impl.cpp
index 72102857cf3..d3084459f69 100644
--- a/src/mongo/db/catalog/index_catalog_entry_impl.cpp
+++ b/src/mongo/db/catalog/index_catalog_entry_impl.cpp
@@ -62,12 +62,14 @@ using std::string;
 IndexCatalogEntryImpl::IndexCatalogEntryImpl(OperationContext* const opCtx,
                                              const std::string& ident,
                                              std::unique_ptr<IndexDescriptor> descriptor,
-                                             CollectionQueryInfo* const queryInfo)
+                                             CollectionQueryInfo* const queryInfo,
+                                             bool isFrozen)
     : _ident(ident),
       _descriptor(std::move(descriptor)),
       _queryInfo(queryInfo),
       _ordering(Ordering::make(_descriptor->keyPattern())),
       _isReady(false),
+      _isFrozen(isFrozen),
       _isDropped(false),
       _prefix(DurableCatalog::get(opCtx)->getIndexPrefix(
           opCtx, _descriptor->getCollection()->getCatalogId(), _descriptor->indexName())) {
@@ -145,6 +147,11 @@ bool IndexCatalogEntryImpl::isReady(OperationContext* opCtx) const {
     return _isReady;
 }
 
+bool IndexCatalogEntryImpl::isFrozen() const {
+    invariant(!_isFrozen || !_isReady);
+    return _isFrozen;
+}
+
 bool IndexCatalogEntryImpl::isMultikey() const {
     return _isMultikey.load();
 }
diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.h b/src/mongo/db/catalog/index_catalog_entry_impl.h
index 243bd412f38..7a026b4bfd9 100644
--- a/src/mongo/db/catalog/index_catalog_entry_impl.h
+++ b/src/mongo/db/catalog/index_catalog_entry_impl.h
@@ -60,7 +60,8 @@ public:
     IndexCatalogEntryImpl(OperationContext* opCtx,
                           const std::string& ident,
                           std::unique_ptr<IndexDescriptor> descriptor,  // ownership passes to me
-                          CollectionQueryInfo* queryInfo);              // not owned, optional
+                          CollectionQueryInfo* queryInfo,               // not owned, optional
+                          bool isFrozen);
 
     ~IndexCatalogEntryImpl() final;
 
@@ -164,6 +165,8 @@ public:
     // if this ready is ready for queries
     bool isReady(OperationContext* opCtx) const final;
 
+    bool isFrozen() const final;
+
     KVPrefix getPrefix() const final {
         return _prefix;
     }
@@ -214,8 +217,9 @@ private:
 
     // cached stuff
 
-    Ordering _ordering;           // TODO: this might be b-tree specific
-    bool _isReady;                // cache of NamespaceDetails info
+    Ordering _ordering;  // TODO: this might be b-tree specific
+    bool _isReady;       // cache of NamespaceDetails info
+    bool _isFrozen;
     AtomicWord<bool> _isDropped;  // Whether the index drop is committed.
 
     // Set to true if this index supports path-level multikey tracking.
diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp
index eafde9b1b86..0eaff8e3bf6 100644
--- a/src/mongo/db/catalog/index_catalog_impl.cpp
+++ b/src/mongo/db/catalog/index_catalog_impl.cpp
@@ -66,6 +66,7 @@
 #include "mongo/db/query/collection_query_info.h"
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/repl_set_member_in_standalone_mode.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/storage/durable_catalog.h"
@@ -105,8 +106,6 @@ Status IndexCatalogImpl::init(OperationContext* opCtx) {
         const string& indexName = indexNames[i];
         BSONObj spec =
             durableCatalog->getIndexSpec(opCtx, _collection->getCatalogId(), indexName).getOwned();
-        invariant(durableCatalog->isIndexReady(opCtx, _collection->getCatalogId(), indexName));
-
         BSONObj keyPattern = spec.getObjectField("key");
         auto descriptor =
             std::make_unique<IndexDescriptor>(_collection, _getAccessMethodName(keyPattern), spec);
@@ -115,10 +114,26 @@ Status IndexCatalogImpl::init(OperationContext* opCtx) {
                 .registerTTLInfo(std::make_pair(_collection->uuid(), indexName));
         }
 
-        const bool initFromDisk = true;
-        const bool isReadyIndex = true;
-        IndexCatalogEntry* entry =
-            createIndexEntry(opCtx, std::move(descriptor), initFromDisk, isReadyIndex);
+        // We intentionally do not drop or rebuild unfinished two-phase index builds before
+        // initializing the IndexCatalog when starting a replica set member in standalone mode. This
+        // is because the index build cannot complete until it receives a replicated commit or
+        // abort oplog entry.
+        if (!durableCatalog->isIndexReady(opCtx, _collection->getCatalogId(), indexName)) {
+            invariant(getReplSetMemberInStandaloneMode(opCtx->getServiceContext()));
+            auto buildUUID =
+                durableCatalog->getIndexBuildUUID(opCtx, _collection->getCatalogId(), indexName);
+            invariant(buildUUID);
+
+            // Indicate that this index is "frozen". It is not ready but is not currently in
+            // progress either. These indexes may be dropped.
+            auto flags = CreateIndexEntryFlags::kInitFromDisk | CreateIndexEntryFlags::kFrozen;
+            IndexCatalogEntry* entry = createIndexEntry(opCtx, std::move(descriptor), flags);
+            fassert(31433, !entry->isReady(opCtx));
+            continue;
+        }
+
+        auto flags = CreateIndexEntryFlags::kInitFromDisk | CreateIndexEntryFlags::kIsReady;
+        IndexCatalogEntry* entry = createIndexEntry(opCtx, std::move(descriptor), flags);
 
         fassert(17340, entry->isReady(opCtx));
     }
@@ -346,8 +361,7 @@ std::vector<BSONObj> IndexCatalogImpl::removeExistingIndexes(
 
 IndexCatalogEntry* IndexCatalogImpl::createIndexEntry(OperationContext* opCtx,
                                                       std::unique_ptr<IndexDescriptor> descriptor,
-                                                      bool initFromDisk,
-                                                      bool isReadyIndex) {
+                                                      CreateIndexEntryFlags flags) {
     Status status = _isSpecOk(opCtx, descriptor->infoObj());
     if (!status.isOK()) {
         severe() << "Found an invalid index " << descriptor->infoObj() << " on the "
@@ -359,9 +373,13 @@ IndexCatalogEntry* IndexCatalogImpl::createIndexEntry(OperationContext* opCtx,
     std::string ident = engine->getCatalog()->getIndexIdent(
         opCtx, _collection->getCatalogId(), descriptor->indexName());
 
+    bool isReadyIndex = CreateIndexEntryFlags::kIsReady & flags;
+    bool frozen = CreateIndexEntryFlags::kFrozen & flags;
+    invariant(!frozen || !isReadyIndex);
+
     auto* const descriptorPtr = descriptor.get();
     auto entry = std::make_shared<IndexCatalogEntryImpl>(
-        opCtx, ident, std::move(descriptor), &CollectionQueryInfo::get(_collection));
+        opCtx, ident, std::move(descriptor), &CollectionQueryInfo::get(_collection), frozen);
 
     IndexDescriptor* desc = entry->descriptor();
 
@@ -373,6 +391,7 @@ IndexCatalogEntry* IndexCatalogImpl::createIndexEntry(OperationContext* opCtx,
 
     entry->init(std::move(accessMethod));
 
+
     IndexCatalogEntry* save = entry.get();
     if (isReadyIndex) {
         _readyIndexes.add(std::move(entry));
@@ -380,6 +399,7 @@ IndexCatalogEntry* IndexCatalogImpl::createIndexEntry(OperationContext* opCtx,
         _buildingIndexes.add(std::move(entry));
     }
 
+    bool initFromDisk = CreateIndexEntryFlags::kInitFromDisk & flags;
     if (!initFromDisk &&
         UncommittedCollections::getForTxn(opCtx, descriptorPtr->parentNS()) == nullptr) {
         opCtx->recoveryUnit()->onRollback([this, opCtx, isReadyIndex, descriptor = descriptorPtr] {
@@ -746,6 +766,17 @@ Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx,
                               str::stream() << "Index with name: " << name
                                             << " already exists with different options");
 
+
+            // If an identical index exists, but it is frozen, return an error with a different
+            // code to the user, forcing the user to drop before recreating the index.
+            auto entry = getEntry(desc);
+            if (entry->isFrozen()) {
+                return Status(ErrorCodes::CannotCreateIndex,
+                              str::stream() << "An identical, unfinished index already exists. The "
+                                               "index must be dropped first: "
+                                            << name << ", spec: " << desc->infoObj());
+            }
+
             // Index already exists with the same options, so no need to build a new
             // one (not an error). Most likely requested by a client using ensureIndex.
             return Status(ErrorCodes::IndexAlreadyExists,
@@ -1233,10 +1264,8 @@ const IndexDescriptor* IndexCatalogImpl::refreshEntry(OperationContext* opCtx,
     // to the CollectionQueryInfo.
     auto newDesc =
         std::make_unique<IndexDescriptor>(_collection, _getAccessMethodName(keyPattern), spec);
-    const bool initFromDisk = false;
-    const bool isReadyIndex = true;
     const IndexCatalogEntry* newEntry =
-        createIndexEntry(opCtx, std::move(newDesc), initFromDisk, isReadyIndex);
+        createIndexEntry(opCtx, std::move(newDesc), CreateIndexEntryFlags::kIsReady);
     invariant(newEntry->isReady(opCtx));
     CollectionQueryInfo::get(_collection).addedIndex(opCtx, newEntry->descriptor());
 
diff --git a/src/mongo/db/catalog/index_catalog_impl.h b/src/mongo/db/catalog/index_catalog_impl.h
index df13439ffe4..36709dba180 100644
--- a/src/mongo/db/catalog/index_catalog_impl.h
+++ b/src/mongo/db/catalog/index_catalog_impl.h
@@ -173,8 +173,7 @@ public:
 
     IndexCatalogEntry* createIndexEntry(OperationContext* opCtx,
                                         std::unique_ptr<IndexDescriptor> descriptor,
-                                        bool initFromDisk,
-                                        bool isReadyIndex) override;
+                                        CreateIndexEntryFlags flags) override;
 
     /**
      * Call this only on an empty collection from inside a WriteUnitOfWork. Index creation on an
diff --git a/src/mongo/db/catalog/index_catalog_noop.h b/src/mongo/db/catalog/index_catalog_noop.h
index f6806353ebf..ebe91ec2b72 100644
--- a/src/mongo/db/catalog/index_catalog_noop.h
+++ b/src/mongo/db/catalog/index_catalog_noop.h
@@ -131,8 +131,7 @@ public:
 
     IndexCatalogEntry* createIndexEntry(OperationContext* opCtx,
                                         std::unique_ptr<IndexDescriptor> descriptor,
-                                        bool initFromDisk,
-                                        bool isReadyIndex) override {
+                                        CreateIndexEntryFlags flags) override {
         return nullptr;
     }
 
diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp
index 3897bd82cd4..abbccedf1dd 100644
--- a/src/mongo/db/repair_database_and_check_version.cpp
+++ b/src/mongo/db/repair_database_and_check_version.cpp
@@ -316,6 +316,15 @@ void rebuildIndexes(OperationContext* opCtx, StorageEngine* storageEngine) {
         fassert(40592, rebuildIndexesOnCollection(opCtx, collection, indexSpecs, RepairData::kNo));
     }
 
+
+    // Two-phase index builds depend on a replicated 'commitIndexBuild' oplog entry to commit.
+    // Therefore, when a replica set member is started in standalone mode, we cannot restart the
+    // index build.
+    if (getReplSetMemberInStandaloneMode(opCtx->getServiceContext())) {
+        log() << "Not restarting unfinished index builds because we are in standalone mode";
+        return;
+    }
+
     // Once all unfinished indexes have been rebuilt, restart any unfinished index builds. This will
     // not build any indexes to completion, but rather start the background thread to build the
     // index, and wait for a replicated commit or abort oplog entry.