SERVER-16274 synchronize start of bg index builds on secondaries

3 files changed, 42 insertions, 0 deletions

diff --git a/src/mongo/db/index_builder.cpp b/src/mongo/db/index_builder.cpp
index 92b4c404150..5a064e57877 100644
--- a/src/mongo/db/index_builder.cpp
+++ b/src/mongo/db/index_builder.cpp
@@ -31,6 +31,7 @@
 #include "mongo/db/client.h"
 #include "mongo/db/curop.h"
 #include "mongo/db/catalog/database.h"
+#include "mongo/db/db.h"
 #include "mongo/db/d_concurrency.h"
 #include "mongo/db/repl/rs.h"
 #include "mongo/util/mongoutils/str.h"
@@ -39,6 +40,22 @@ namespace mongo {
 
     AtomicUInt IndexBuilder::_indexBuildCount = 0;
 
+namespace {
+    // Synchronization tools when replication spawns a background index in a new thread.
+    // The bool is 'true' when a new background index has started in a new thread but the
+    // parent thread has not yet synchronized with it.
+    bool _bgIndexStarting(false);
+    boost::mutex _bgIndexStartingMutex;
+    boost::condition_variable _bgIndexStartingCondVar;
+
+    void _setBgIndexStarting() {
+        boost::mutex::scoped_lock lk(_bgIndexStartingMutex);
+        invariant(_bgIndexStarting == false);
+        _bgIndexStarting = true;
+        _bgIndexStartingCondVar.notify_one();
+    }
+} // namespace
+
     IndexBuilder::IndexBuilder(const BSONObj& index) :
         BackgroundJob(true /* self-delete */), _index(index.getOwned()),
         _name(str::stream() << "repl index builder " << (_indexBuildCount++).get()) {
@@ -62,6 +79,10 @@ namespace mongo {
         NamespaceString ns(_index["ns"].String());
         Client::WriteContext ctx(ns.getSystemIndexesCollection());
 
+        // Show which index we're background building in the curop display, and unblock waiters.
+        cc().curop()->setQuery(_index);
+        _setBgIndexStarting();
+
         Status status = build( ctx.ctx() );
         if ( !status.isOK() ) {
             log() << "IndexBuilder could not build index: " << status.toString();
@@ -90,6 +111,15 @@ namespace mongo {
         return status;
     }
 
+    void IndexBuilder::waitForBgIndexStarting() {
+        boost::unique_lock<boost::mutex> lk(_bgIndexStartingMutex);
+        while (_bgIndexStarting == false) {
+            _bgIndexStartingCondVar.wait(lk);
+        }
+        // Reset for next time.
+        _bgIndexStarting = false;
+    }
+
     std::vector<BSONObj> 
     IndexBuilder::killMatchingIndexBuilds(Collection* collection,
                                           const IndexCatalog::IndexKillCriteria& criteria) {
@@ -103,6 +133,8 @@ namespace mongo {
             IndexBuilder* indexBuilder = new IndexBuilder(indexes[i]);
             // This looks like a memory leak, but indexBuilder deletes itself when it finishes
             indexBuilder->go();
+            dbtemprelease release;
+            IndexBuilder::waitForBgIndexStarting();
         }
     }
 }
diff --git a/src/mongo/db/index_builder.h b/src/mongo/db/index_builder.h
index c3d042b5cea..6730d98ecf3 100644
--- a/src/mongo/db/index_builder.h
+++ b/src/mongo/db/index_builder.h
@@ -72,6 +72,13 @@ namespace mongo {
          */
         static void restoreIndexes(const std::vector<BSONObj>& indexes);
 
+        /**
+         * Waits for a background index build to register itself.  This function must be called
+         * after starting a background index build via a BackgroundJob and before starting a
+         * subsequent one.
+         */
+        static void waitForBgIndexStarting();
+
     private:
         const BSONObj _index;
         std::string _name; // name of this builder, not related to the index
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index 2c5dcbc2043..0e3b31ac097 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -526,6 +526,9 @@ namespace mongo {
                     IndexBuilder* builder = new IndexBuilder(o);
                     // This spawns a new thread and returns immediately.
                     builder->go();
+                    // Wait for thread to start and register itself
+                    dbtemprelease release;
+                    IndexBuilder::waitForBgIndexStarting();
                 }
                 else {
                     Client::Context* ctx = cc().getContext();