SERVER-16274 synchronize start of bg index builds on secondaries

1 files changed, 38 insertions, 5 deletions

diff --git a/src/mongo/db/index_builder.cpp b/src/mongo/db/index_builder.cpp
index 8e7d38fd8a3..95413374f4b 100644
--- a/src/mongo/db/index_builder.cpp
+++ b/src/mongo/db/index_builder.cpp
@@ -47,6 +47,22 @@ namespace mongo {
 
     AtomicUInt32 IndexBuilder::_indexBuildCount;
 
+namespace {
+    // Synchronization tools when replication spawns a background index in a new thread.
+    // The bool is 'true' when a new background index has started in a new thread but the
+    // parent thread has not yet synchronized with it.
+    bool _bgIndexStarting(false);
+    boost::mutex _bgIndexStartingMutex;
+    boost::condition_variable _bgIndexStartingCondVar;
+
+    void _setBgIndexStarting() {
+        boost::mutex::scoped_lock lk(_bgIndexStartingMutex);
+        invariant(_bgIndexStarting == false);
+        _bgIndexStarting = true;
+        _bgIndexStartingCondVar.notify_one();
+    }
+} // namespace
+
     IndexBuilder::IndexBuilder(const BSONObj& index) :
         BackgroundJob(true /* self-delete */), _index(index.getOwned()),
         _name(str::stream() << "repl index builder " << _indexBuildCount.addAndFetch(1)) {
@@ -90,6 +106,15 @@ namespace mongo {
         return _build(txn, db, false, NULL);
     }
 
+    void IndexBuilder::waitForBgIndexStarting() {
+        boost::unique_lock<boost::mutex> lk(_bgIndexStartingMutex);
+        while (_bgIndexStarting == false) {
+            _bgIndexStartingCondVar.wait(lk);
+        }
+        // Reset for next time.
+        _bgIndexStarting = false;
+    }
+
     Status IndexBuilder::_build(OperationContext* txn,
                                 Database* db,
                                 bool allowBackgroundBuilding,
@@ -109,6 +134,7 @@ namespace mongo {
 
         MultiIndexBlock indexer(txn, c);
         indexer.allowInterruption();
+
         if (allowBackgroundBuilding)
             indexer.allowBackgroundBuilding();
 
@@ -116,12 +142,18 @@ namespace mongo {
         IndexDescriptor* descriptor(NULL);
         try {
             status = indexer.init(_index);
-            if ( status.code() == ErrorCodes::IndexAlreadyExists )
+            if ( status.code() == ErrorCodes::IndexAlreadyExists ) {
+                if (allowBackgroundBuilding) {
+                    // Must set this in case anyone is waiting for this build.
+                    _setBgIndexStarting();
+                }
                 return Status::OK();
+            }
 
             if (status.isOK()) {
                 if (allowBackgroundBuilding) {
                     descriptor = indexer.registerIndexBuild();
+                    _setBgIndexStarting();
                     invariant(dbLock);
                     dbLock->relockWithMode(MODE_IX);
                 }
@@ -159,20 +191,21 @@ namespace mongo {
         return status;
     }
 
-    std::vector<BSONObj> 
+    std::vector<BSONObj>
     IndexBuilder::killMatchingIndexBuilds(Collection* collection,
                                           const IndexCatalog::IndexKillCriteria& criteria) {
         invariant(collection);
         return collection->getIndexCatalog()->killMatchingIndexBuilds(criteria);
     }
 
-    void IndexBuilder::restoreIndexes(const std::vector<BSONObj>& indexes) {
-        log() << "restarting " << indexes.size() << " index build(s)" << endl;
+    void IndexBuilder::restoreIndexes(OperationContext* txn, const std::vector<BSONObj>& indexes) {
+        log() << "restarting " << indexes.size() << " background index build(s)" << endl;
         for (int i = 0; i < static_cast<int>(indexes.size()); i++) {
             IndexBuilder* indexBuilder = new IndexBuilder(indexes[i]);
             // This looks like a memory leak, but indexBuilder deletes itself when it finishes
             indexBuilder->go();
+            Lock::TempRelease release(txn->lockState());
+            IndexBuilder::waitForBgIndexStarting();
         }
     }
 }
-