SERVER-40023 remove background index support from IndexBuilder

18 files changed, 10 insertions, 191 deletions

diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index ce21afed7a8..f91db1bebe9 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -1059,6 +1059,7 @@ env.Library(
     LIBDEPS_PRIVATE=[
         "logical_clock",
         "server_options_core",
+        '$BUILD_DIR/mongo/db/catalog/index_timestamp_helper',
         '$BUILD_DIR/mongo/db/repl/timestamp_block',
     ],
 )
diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp
index 48d10844bf8..2ddae23aa80 100644
--- a/src/mongo/db/catalog/capped_utils.cpp
+++ b/src/mongo/db/catalog/capped_utils.cpp
@@ -45,7 +45,6 @@
 #include "mongo/db/client.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/curop.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/query/plan_yield_policy.h"
diff --git a/src/mongo/db/catalog/drop_collection.cpp b/src/mongo/db/catalog/drop_collection.cpp
index 594328bddfd..93b281930a1 100644
--- a/src/mongo/db/catalog/drop_collection.cpp
+++ b/src/mongo/db/catalog/drop_collection.cpp
@@ -39,7 +39,6 @@
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/service_context.h"
diff --git a/src/mongo/db/catalog/drop_indexes.cpp b/src/mongo/db/catalog/drop_indexes.cpp
index 03b21ef9801..eb8049c3090 100644
--- a/src/mongo/db/catalog/drop_indexes.cpp
+++ b/src/mongo/db/catalog/drop_indexes.cpp
@@ -40,7 +40,6 @@
 #include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/service_context.h"
diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp
index 01dffe8848a..fcf5d8065ce 100644
--- a/src/mongo/db/catalog/rename_collection.cpp
+++ b/src/mongo/db/catalog/rename_collection.cpp
@@ -45,7 +45,6 @@
 #include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/ops/insert.h"
diff --git a/src/mongo/db/cloner.cpp b/src/mongo/db/cloner.cpp
index f93eaa1ceb6..ddceac82399 100644
--- a/src/mongo/db/cloner.cpp
+++ b/src/mongo/db/cloner.cpp
@@ -53,7 +53,6 @@
 #include "mongo/db/curop.h"
 #include "mongo/db/dbdirectclient.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/op_observer.h"
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index c85c1edce37..fd1fc79cdae 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -371,7 +371,6 @@ env.Library(
         '$BUILD_DIR/mongo/db/dbhelpers',
         '$BUILD_DIR/mongo/db/exec/stagedebug_cmd',
         '$BUILD_DIR/mongo/db/index_builds_coordinator_interface',
-        '$BUILD_DIR/mongo/db/index_d',
         '$BUILD_DIR/mongo/db/repl/dbcheck',
         '$BUILD_DIR/mongo/db/repl/oplog',
         '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
diff --git a/src/mongo/db/commands/clone_collection.cpp b/src/mongo/db/commands/clone_collection.cpp
index 36c752541ea..b0eeddefe5b 100644
--- a/src/mongo/db/commands/clone_collection.cpp
+++ b/src/mongo/db/commands/clone_collection.cpp
@@ -43,7 +43,6 @@
 #include "mongo/db/cloner.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/rename_collection.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/ops/insert.h"
diff --git a/src/mongo/db/commands/collection_to_capped.cpp b/src/mongo/db/commands/collection_to_capped.cpp
index a57b5d66c88..e2fab366906 100644
--- a/src/mongo/db/commands/collection_to_capped.cpp
+++ b/src/mongo/db/commands/collection_to_capped.cpp
@@ -35,7 +35,6 @@
 #include "mongo/db/client.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/db_raii.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/query/find.h"
 #include "mongo/db/query/internal_plans.h"
diff --git a/src/mongo/db/commands/compact.cpp b/src/mongo/db/commands/compact.cpp
index 8e74b1d8e46..9ae2e824099 100644
--- a/src/mongo/db/commands/compact.cpp
+++ b/src/mongo/db/commands/compact.cpp
@@ -43,7 +43,6 @@
 #include "mongo/db/concurrency/d_concurrency.h"
 #include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/views/view_catalog.h"
diff --git a/src/mongo/db/commands/dbcommands.cpp b/src/mongo/db/commands/dbcommands.cpp
index 0aee8741361..7378c1fae2c 100644
--- a/src/mongo/db/commands/dbcommands.cpp
+++ b/src/mongo/db/commands/dbcommands.cpp
@@ -65,7 +65,6 @@
 #include "mongo/db/exec/working_set_common.h"
 #include "mongo/db/index/index_access_method.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/introspect.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/json.h"
diff --git a/src/mongo/db/commands/dbcommands_d.cpp b/src/mongo/db/commands/dbcommands_d.cpp
index 0cbf9cb0cbe..a21e2d76c47 100644
--- a/src/mongo/db/commands/dbcommands_d.cpp
+++ b/src/mongo/db/commands/dbcommands_d.cpp
@@ -66,7 +66,6 @@
 #include "mongo/db/exec/working_set_common.h"
 #include "mongo/db/index/index_access_method.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/introspect.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/json.h"
diff --git a/src/mongo/db/commands/drop_indexes.cpp b/src/mongo/db/commands/drop_indexes.cpp
index 3d189e8472a..1e8ac1b4a25 100644
--- a/src/mongo/db/commands/drop_indexes.cpp
+++ b/src/mongo/db/commands/drop_indexes.cpp
@@ -48,7 +48,6 @@
 #include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/logical_clock.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/service_context.h"
diff --git a/src/mongo/db/commands/rename_collection_cmd.cpp b/src/mongo/db/commands/rename_collection_cmd.cpp
index 4d2b5ec488f..fe2a048b153 100644
--- a/src/mongo/db/commands/rename_collection_cmd.cpp
+++ b/src/mongo/db/commands/rename_collection_cmd.cpp
@@ -40,7 +40,6 @@
 #include "mongo/db/commands/rename_collection.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/ops/insert.h"
diff --git a/src/mongo/db/commands/test_commands.cpp b/src/mongo/db/commands/test_commands.cpp
index 46016312447..99c3b94f1e7 100644
--- a/src/mongo/db/commands/test_commands.cpp
+++ b/src/mongo/db/commands/test_commands.cpp
@@ -41,7 +41,6 @@
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/test_commands_enabled.h"
 #include "mongo/db/db_raii.h"
-#include "mongo/db/index_builder.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/service_context.h"
diff --git a/src/mongo/db/index_builder.cpp b/src/mongo/db/index_builder.cpp
index 14eb136e58a..2302620a18e 100644
--- a/src/mongo/db/index_builder.cpp
+++ b/src/mongo/db/index_builder.cpp
@@ -34,16 +34,13 @@
 #include "mongo/db/index_builder.h"
 
 #include "mongo/bson/bsonobjbuilder.h"
-#include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/catalog/database.h"
 #include "mongo/db/catalog/database_holder.h"
 #include "mongo/db/catalog/index_timestamp_helper.h"
 #include "mongo/db/catalog/multi_index_block.h"
-#include "mongo/db/client.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
-#include "mongo/db/logical_clock.h"
 #include "mongo/db/op_observer.h"
 #include "mongo/db/repl/timestamp_block.h"
 #include "mongo/db/server_options.h"
@@ -53,8 +50,6 @@
 
 namespace mongo {
 
-using std::endl;
-
 AtomicWord<unsigned> IndexBuilder::_indexBuildCount;
 
 namespace {
@@ -62,27 +57,13 @@ namespace {
 const StringData kIndexesFieldName = "indexes"_sd;
 const StringData kCommandName = "createIndexes"_sd;
 
-// Synchronization tools when replication spawns a background index in a new thread.
-// The bool is 'true' when a new background index has started in a new thread but the
-// parent thread has not yet synchronized with it.
-bool _bgIndexStarting(false);
-stdx::mutex _bgIndexStartingMutex;
-stdx::condition_variable _bgIndexStartingCondVar;
-
-void _setBgIndexStarting() {
-    stdx::lock_guard<stdx::mutex> lk(_bgIndexStartingMutex);
-    invariant(_bgIndexStarting == false);
-    _bgIndexStarting = true;
-    _bgIndexStartingCondVar.notify_one();
-}
 }  // namespace
 
 IndexBuilder::IndexBuilder(const BSONObj& index,
                            IndexConstraints indexConstraints,
                            ReplicatedWrites replicatedWrites,
                            Timestamp initIndexTs)
-    : BackgroundJob(true /* self-delete */),
-      _index(index.getOwned()),
+    : _index(index.getOwned()),
       _indexConstraints(indexConstraints),
       _replicatedWrites(replicatedWrites),
       _initIndexTs(initIndexTs),
@@ -98,64 +79,17 @@ std::string IndexBuilder::name() const {
     return _name;
 }
 
-void IndexBuilder::run() {
-    ThreadClient tc(name(), getGlobalServiceContext());
-    LOG(2) << "IndexBuilder building index " << _index;
-
-    auto opCtx = cc().makeOperationContext();
-    ShouldNotConflictWithSecondaryBatchApplicationBlock shouldNotConflictBlock(opCtx->lockState());
-
-    // If the calling thread is not replicating writes, neither should this thread.
-    boost::optional<repl::UnreplicatedWritesBlock> unreplicatedWrites;
-    if (_replicatedWrites == ReplicatedWrites::kUnreplicated) {
-        unreplicatedWrites.emplace(opCtx.get());
-    }
-
-    AuthorizationSession::get(opCtx->getClient())->grantInternalAuthorization(opCtx.get());
-
-    {
-        stdx::lock_guard<Client> lk(*(opCtx->getClient()));
-        CurOp::get(opCtx.get())->setNetworkOp_inlock(dbInsert);
-    }
-    NamespaceString ns(_index["ns"].String());
-
-    Lock::DBLock dlk(opCtx.get(), ns.db(), MODE_X);
-    auto databaseHolder = DatabaseHolder::get(opCtx.get());
-    auto db = databaseHolder->getDb(opCtx.get(), ns.db().toString());
-
-    // This background index build can only be interrupted at shutdown.  For the duration of the
-    // OperationContext::runWithoutInterruptionExceptAtGlobalShutdown() invocation, any kill status
-    // set by the killOp command will be ignored.  After
-    // OperationContext::runWithoutInterruptionExceptAtGlobalShutdown() returns, any call to
-    // OperationContext::checkForInterrupt() will see the kill status and respond accordingly
-    // (checkForInterrupt() will throw an exception while checkForInterruptNoAssert() returns an
-    // error Status).
-    Status status = opCtx->runWithoutInterruptionExceptAtGlobalShutdown([&, this] {
-        return _buildAndHandleErrors(opCtx.get(), db, true /* buildInBackground */, &dlk);
-    });
-    if (!status.isOK()) {
-        error() << "IndexBuilder could not build index: " << redact(status);
-        fassert(28555, ErrorCodes::isInterruption(status.code()));
-    }
-}
-
 Status IndexBuilder::buildInForeground(OperationContext* opCtx, Database* db) const {
     return _buildAndHandleErrors(opCtx, db, false /*buildInBackground */, nullptr);
 }
 
-void IndexBuilder::waitForBgIndexStarting() {
-    stdx::unique_lock<stdx::mutex> lk(_bgIndexStartingMutex);
-    while (_bgIndexStarting == false) {
-        _bgIndexStartingCondVar.wait(lk);
-    }
-    // Reset for next time.
-    _bgIndexStarting = false;
-}
-
 Status IndexBuilder::_buildAndHandleErrors(OperationContext* opCtx,
                                            Database* db,
                                            bool buildInBackground,
                                            Lock::DBLock* dbLock) const {
+    invariant(!buildInBackground);
+    invariant(!dbLock);
+
     const NamespaceString ns(_index["ns"].String());
 
     Collection* coll = db->getCollection(opCtx, ns);
@@ -167,30 +101,7 @@ Status IndexBuilder::_buildAndHandleErrors(OperationContext* opCtx,
     // The 'indexer' can throw, so ensure build cleanup occurs.
     ON_BLOCK_EXIT([&] { indexer.cleanUpAfterBuild(opCtx, coll); });
 
-    auto status = _build(opCtx, buildInBackground, coll, indexer, dbLock);
-    // Background index builds are not allowed to return errors because they run in a background
-    // thread.
-    if (status.isOK() || !buildInBackground) {
-        invariant(!dbLock || dbLock->mode() == MODE_X);
-        return status;
-    }
-
-    // The MultiIndexBlock destructor may only be called when an X lock is held on the database.
-    if (dbLock->mode() != MODE_X) {
-        UninterruptibleLockGuard noInterrupt(opCtx->lockState());
-        dbLock->relockWithMode(MODE_X);
-    }
-
-    invariant(status.code() != ErrorCodes::WriteConflict);
-
-    if (status.code() == ErrorCodes::InterruptedAtShutdown) {
-        // leave it as-if kill -9 happened. This will be handled on restart.
-        indexer.abortWithoutCleanup(opCtx);
-        return status;
-    }
-
-    error() << "Background index build failed. Status: " << redact(status);
-    fassertFailed(50769);
+    return _build(opCtx, buildInBackground, coll, indexer, dbLock);
 }
 
 Status IndexBuilder::_build(OperationContext* opCtx,
@@ -198,6 +109,9 @@ Status IndexBuilder::_build(OperationContext* opCtx,
                             Collection* coll,
                             MultiIndexBlock& indexer,
                             Lock::DBLock* dbLock) const try {
+    invariant(!buildInBackground);
+    invariant(!dbLock);
+
     auto ns = coll->ns();
 
     {
@@ -242,26 +156,12 @@ Status IndexBuilder::_build(OperationContext* opCtx,
          _indexConstraints == IndexConstraints::kRelax)) {
         LOG(1) << "Ignoring indexing error: " << redact(status);
 
-        // Must set this in case anyone is waiting for this build.
-        if (dbLock) {
-            _setBgIndexStarting();
-        }
         return Status::OK();
     }
     if (!status.isOK()) {
         return status;
     }
 
-    if (buildInBackground) {
-        invariant(dbLock);
-
-        _setBgIndexStarting();
-        opCtx->recoveryUnit()->abandonSnapshot();
-
-        UninterruptibleLockGuard noInterrupt(opCtx->lockState());
-        dbLock->relockWithMode(MODE_IX);
-    }
-
     {
         Lock::CollectionLock collLock(opCtx->lockState(), ns.ns(), MODE_IX);
         // WriteConflict exceptions and statuses are not expected to escape this method.
@@ -271,49 +171,6 @@ Status IndexBuilder::_build(OperationContext* opCtx,
         return status;
     }
 
-    if (buildInBackground) {
-        {
-            // Perform the first drain while holding an intent lock.
-            Lock::CollectionLock collLock(opCtx->lockState(), ns.ns(), MODE_IX);
-
-            // Read at a point in time so that the drain, which will timestamp writes at
-            // lastApplied, can never commit writes earlier than its read timestamp.
-            status = indexer.drainBackgroundWrites(opCtx, RecoveryUnit::ReadSource::kNoOverlap);
-        }
-        if (!status.isOK()) {
-            return status;
-        }
-
-        // Perform the second drain while stopping inserts into the collection.
-        {
-            Lock::CollectionLock colLock(opCtx->lockState(), ns.ns(), MODE_S);
-            status = indexer.drainBackgroundWrites(opCtx);
-        }
-        if (!status.isOK()) {
-            return status;
-        }
-
-        opCtx->recoveryUnit()->abandonSnapshot();
-
-        UninterruptibleLockGuard noInterrupt(opCtx->lockState());
-        dbLock->relockWithMode(MODE_X);
-
-        // Perform the third and final drain after releasing a shared lock and reacquiring an
-        // exclusive lock on the database.
-        status = indexer.drainBackgroundWrites(opCtx);
-        if (!status.isOK()) {
-            return status;
-        }
-
-        // Only perform constraint checking when enforced (on primaries).
-        if (_indexConstraints == IndexConstraints::kEnforce) {
-            status = indexer.checkConstraints(opCtx);
-            if (!status.isOK()) {
-                return status;
-            }
-        }
-    }
-
     status = writeConflictRetry(opCtx, "Commit index build", ns.ns(), [opCtx, coll, &indexer, &ns] {
         WriteUnitOfWork wunit(opCtx);
         auto status = indexer.commit(opCtx,
@@ -335,19 +192,6 @@ Status IndexBuilder::_build(OperationContext* opCtx,
         return status;
     }
 
-    if (buildInBackground) {
-        invariant(opCtx->lockState()->isDbLockedForMode(ns.db(), MODE_X),
-                  str::stream() << "Database not locked in exclusive mode after committing "
-                                   "background index: "
-                                << ns.ns()
-                                << ": "
-                                << _index);
-        auto databaseHolder = DatabaseHolder::get(opCtx);
-        auto reloadDb = databaseHolder->getDb(opCtx, ns.db());
-        fassert(28553, reloadDb);
-        fassert(28554, reloadDb->getCollection(opCtx, ns));
-    }
-
     return Status::OK();
 } catch (const DBException& e) {
     return e.toStatus();
diff --git a/src/mongo/db/index_builder.h b/src/mongo/db/index_builder.h
index e90815ec510..ba2fc769a25 100644
--- a/src/mongo/db/index_builder.h
+++ b/src/mongo/db/index_builder.h
@@ -37,7 +37,6 @@
 #include "mongo/db/concurrency/d_concurrency.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/platform/atomic_word.h"
-#include "mongo/util/background.h"
 
 namespace mongo {
 
@@ -67,7 +66,7 @@ class OperationContext;
  * oplog entries associated with this index build.
  * The argument "initIndexTs" specifies the timestamp to be used to make the initial catalog write.
  */
-class IndexBuilder : public BackgroundJob {
+class IndexBuilder {
 public:
     /**
      * Indicates whether or not to ignore indexing constraints.
@@ -85,8 +84,6 @@ public:
                  Timestamp initIndexTs = Timestamp::min());
     virtual ~IndexBuilder();
 
-    virtual void run();
-
     /**
      * name of the builder, not the index
      */
@@ -97,13 +94,6 @@ public:
      */
     Status buildInForeground(OperationContext* opCtx, Database* db) const;
 
-    /**
-     * Waits for a background index build to register itself.  This function must be called
-     * after starting a background index build via a BackgroundJob and before starting a
-     * subsequent one.
-     */
-    static void waitForBgIndexStarting();
-
     static bool canBuildInBackground();
 
 private:
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index ea5411e16fd..e3ff0a3a6a9 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -143,7 +143,6 @@ env.Library(
         '$BUILD_DIR/mongo/db/concurrency/lock_manager',
         '$BUILD_DIR/mongo/db/db_raii',
         '$BUILD_DIR/mongo/db/dbhelpers',
-        '$BUILD_DIR/mongo/db/index_d',
         '$BUILD_DIR/mongo/db/index/index_descriptor',
         '$BUILD_DIR/mongo/db/query_exec',
         '$BUILD_DIR/mongo/util/fail_point',
@@ -1646,7 +1645,6 @@ env.Library(
         '$BUILD_DIR/mongo/db/concurrency/lock_manager',
         '$BUILD_DIR/mongo/db/curop',
         '$BUILD_DIR/mongo/db/free_mon/free_mon_mongod',
-        '$BUILD_DIR/mongo/db/index_d',
         '$BUILD_DIR/mongo/db/kill_sessions_local',
         '$BUILD_DIR/mongo/db/lasterror',
         '$BUILD_DIR/mongo/db/logical_clock',