SERVER-44250 serialize startIndexBuild oplog write and thread pool scheduling between concurrent threads on primaries

3 files changed, 15 insertions, 0 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index c5d169f7ffe..8ecf15dd47f 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -11219,6 +11219,7 @@ buildvariants:
   - name: compile_TG
     distros:
     - rhel62-large
+  - name: .concurrency !.large !.ubsan !.no_txns !.stepdowns !.kill_terminate !.debug_only
   - name: .jscore .common
   - name: noPassthrough_gen
   - name: noPassthroughWithMongod_gen
diff --git a/src/mongo/db/index_builds_coordinator_mongod.cpp b/src/mongo/db/index_builds_coordinator_mongod.cpp
index 99cdc6b6ca1..2d205a452c4 100644
--- a/src/mongo/db/index_builds_coordinator_mongod.cpp
+++ b/src/mongo/db/index_builds_coordinator_mongod.cpp
@@ -95,6 +95,17 @@ IndexBuildsCoordinatorMongod::startIndexBuild(OperationContext* opCtx,
                                               const UUID& buildUUID,
                                               IndexBuildProtocol protocol,
                                               IndexBuildOptions indexBuildOptions) {
+    // This lock serializes setting up the index build and scheduling it on the thread pool between
+    // concurrent callers. It guarantees that if this thread replicates a "startIndexBuild" oplog
+    // entry in the setup, no other concurrent thread can schedule its work ahead of this one. The
+    // thread pool can become saturated with work such that the task gets queued and not immediately
+    // run. The serialization in this function guarantees that primaries and secondaries will
+    // execute these steps in the same order, which is important to avoid deadlocks due to thread
+    // pool resource contention.
+    // TODO(SERVER-44609): Remove this mutex and setup the index build on the scheduled index build
+    // thread.
+    stdx::unique_lock<Latch> lk(_startBuildMutex);
+
     auto statusWithOptionalResult = _registerAndSetUpIndexBuild(
         opCtx, dbName, collectionUUID, specs, buildUUID, protocol, indexBuildOptions.commitQuorum);
     if (!statusWithOptionalResult.isOK()) {
diff --git a/src/mongo/db/index_builds_coordinator_mongod.h b/src/mongo/db/index_builds_coordinator_mongod.h
index d03e5f33653..46e990abd23 100644
--- a/src/mongo/db/index_builds_coordinator_mongod.h
+++ b/src/mongo/db/index_builds_coordinator_mongod.h
@@ -123,6 +123,9 @@ private:
      */
     void _refreshReplStateFromPersisted(OperationContext* opCtx, const UUID& buildUUID);
 
+    // Serializes setting up an index build and scheduling it on the thread pool.
+    mutable Mutex _startBuildMutex = MONGO_MAKE_LATCH("IndexBuildsCoordinator::_startBuildMutex");
+
     // Thread pool on which index builds are run.
     ThreadPool _threadPool;
 };