SERVER-13951 Split index building in to UnitOfWork-sized stages

All index builds now go through the MultiIndexBuilder as its API was already
close to ideal. The following tickets have also been addressed by this commit:

SERVER-14710 Remove dropDups
SERVER-12309 Cloner build indexes in parallel
SERVER-14737 Initial sync uses bg index building
SERVER-9135 fast index build for initial sync
SERVER-2747 can't kill index in phase 2
SERVER-8917 check error code rather than assuming all errors are dups
SERVER-14820 compact enforces unique while claiming not to
SERVER-14746 IndexRebuilder should be foreground and fail fatally

1 files changed, 79 insertions, 69 deletions

diff --git a/src/mongo/db/index_rebuilder.cpp b/src/mongo/db/index_rebuilder.cpp
index 328a5b62078..0445bdc50c6 100644
--- a/src/mongo/db/index_rebuilder.cpp
+++ b/src/mongo/db/index_rebuilder.cpp
@@ -32,11 +32,15 @@
 
 #include "mongo/db/index_rebuilder.h"
 
+#include <list>
+#include <string>
+
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/auth/user_name.h"
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/database.h"
 #include "mongo/db/catalog/database_catalog_entry.h"
+#include "mongo/db/catalog/index_create.h"
 #include "mongo/db/client.h"
 #include "mongo/db/global_environment_experiment.h"
 #include "mongo/db/instance.h"
@@ -47,46 +51,8 @@
 
 namespace mongo {
 
-    IndexRebuilder indexRebuilder;
-
-    IndexRebuilder::IndexRebuilder() {}
-
-    std::string IndexRebuilder::name() const {
-        return "IndexRebuilder";
-    }
-
-    void IndexRebuilder::run() {
-        Client::initThread(name().c_str()); 
-        ON_BLOCK_EXIT_OBJ(cc(), &Client::shutdown);
-
-        OperationContextImpl txn;
-
-        cc().getAuthorizationSession()->grantInternalAuthorization();
-
-        std::vector<std::string> dbNames;
-
-        StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine();
-        storageEngine->listDatabases( &dbNames );
-
-        try {
-            std::list<std::string> collNames;
-            for (std::vector<std::string>::const_iterator dbName = dbNames.begin();
-                 dbName < dbNames.end();
-                 dbName++) {
-                Client::ReadContext ctx(&txn, *dbName);
-
-                Database* db = ctx.ctx().db();
-                db->getDatabaseCatalogEntry()->getCollectionNamespaces(&collNames);
-            }
-            checkNS(&txn, collNames);
-        }
-        catch (const DBException& e) {
-            warning() << "Index rebuilding did not complete: " << e.what() << endl;
-        }
-        LOG(1) << "checking complete" << endl;
-    }
-
-    void IndexRebuilder::checkNS(OperationContext* txn, const std::list<std::string>& nsToCheck) {
+namespace {
+    void checkNS(OperationContext* txn, const std::list<std::string>& nsToCheck) {
         bool firstTime = true;
         for (std::list<std::string>::const_iterator it = nsToCheck.begin();
                 it != nsToCheck.end();
@@ -98,9 +64,10 @@ namespace mongo {
 
             // This write lock is held throughout the index building process
             // for this namespace.
-            Client::WriteContext ctx(txn, ns);
+            Lock::DBWrite lk(txn->lockState(), ns);
+            Client::Context ctx(txn, ns);
 
-            Collection* collection = ctx.ctx().db()->getCollection(txn, ns);
+            Collection* collection = ctx.db()->getCollection(txn, ns);
             if ( collection == NULL )
                 continue;
 
@@ -112,45 +79,88 @@ namespace mongo {
                 continue;
             }
 
-            vector<BSONObj> indexesToBuild = indexCatalog->getAndClearUnfinishedIndexes(txn);
 
-            // The indexes have now been removed from system.indexes, so the only record is
-            // in-memory. If there is a journal commit between now and when insert() rewrites
-            // the entry and the db crashes before the new system.indexes entry is journalled,
-            // the index will be lost forever.  Thus, we're assuming no journaling will happen
-            // between now and the entry being re-written.
+            MultiIndexBlock indexer(txn, collection);
 
-            if ( indexesToBuild.size() == 0 ) {
-                continue;
-            }
+            {
+                WriteUnitOfWork wunit(txn->recoveryUnit());
+                vector<BSONObj> indexesToBuild = indexCatalog->getAndClearUnfinishedIndexes(txn);
 
-            log() << "found " << indexesToBuild.size()
-                  << " interrupted index build(s) on " << ns;
+                // The indexes have now been removed from system.indexes, so the only record is
+                // in-memory. If there is a journal commit between now and when insert() rewrites
+                // the entry and the db crashes before the new system.indexes entry is journalled,
+                // the index will be lost forever. Thus, we must stay in the same WriteUnitOfWork
+                // to ensure that no journaling will happen between now and the entry being
+                // re-written in MultiIndexBlock::init(). The actual index building is done outside
+                // of this WUOW.
 
-            if (firstTime) {
-                log() << "note: restart the server with --noIndexBuildRetry to skip index rebuilds";
-                firstTime = false;
+                if (indexesToBuild.empty()) {
+                    continue;
+                }
+
+                log() << "found " << indexesToBuild.size()
+                      << " interrupted index build(s) on " << ns;
+
+                if (firstTime) {
+                    log() << "note: restart the server with --noIndexBuildRetry "
+                          << "to skip index rebuilds";
+                    firstTime = false;
+                }
+
+                if (!serverGlobalParams.indexBuildRetry) {
+                    log() << "  not rebuilding interrupted indexes";
+                    continue;
+                }
+
+                uassertStatusOK(indexer.init(indexesToBuild));
+
+                wunit.commit();
             }
 
-            if (!serverGlobalParams.indexBuildRetry) {
-                log() << "  not rebuilding interrupted indexes";
-                continue;
+            try {
+                uassertStatusOK(indexer.insertAllDocumentsInCollection());
+
+                WriteUnitOfWork wunit(txn->recoveryUnit());
+                indexer.commit();
+                wunit.commit();
+            }
+            catch (...) {
+                // If anything went wrong, leave the indexes partially built so that we pick them up
+                // again on restart.
+                indexer.abortWithoutCleanup();
+                throw;
             }
+        }
+    }
+} // namespace
 
-            // TODO: these can/should/must be done in parallel
-            for ( size_t i = 0; i < indexesToBuild.size(); i++ ) {
-                BSONObj indexObj = indexesToBuild[i];
+    void restartInProgressIndexesFromLastShutdown() {
+        OperationContextImpl txn;
 
-                log() << "going to rebuild: " << indexObj;
+        cc().getAuthorizationSession()->grantInternalAuthorization();
 
-                Status status = indexCatalog->createIndex(txn, indexObj, false);
-                if ( !status.isOK() ) {
-                    log() << "building index failed: " << status.toString() << " index: " << indexObj;
-                }
+        std::vector<std::string> dbNames;
 
+        StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine();
+        storageEngine->listDatabases( &dbNames );
+
+        try {
+            std::list<std::string> collNames;
+            for (std::vector<std::string>::const_iterator dbName = dbNames.begin();
+                 dbName < dbNames.end();
+                 ++dbName) {
+                Client::ReadContext ctx(&txn, *dbName);
+
+                Database* db = ctx.ctx().db();
+                db->getDatabaseCatalogEntry()->getCollectionNamespaces(&collNames);
             }
-            ctx.commit();
+            checkNS(&txn, collNames);
         }
+        catch (const DBException& e) {
+            error() << "Index rebuilding did not complete: " << e.toString();
+            log() << "note: restart the server with --noIndexBuildRetry to skip index rebuilds";
+            fassertFailedNoTrace(18643);
+        }
+        LOG(1) << "checking complete" << endl;
     }
-
 }