SERVER-49244 Test that resumable index builds write state to disk on clean shutdown

11 files changed, 318 insertions, 59 deletions

diff --git a/jstests/noPassthrough/hybrid_index_with_updates.js b/jstests/noPassthrough/hybrid_index_with_updates.js
index 727ad643f64..56439d1e594 100644
--- a/jstests/noPassthrough/hybrid_index_with_updates.js
+++ b/jstests/noPassthrough/hybrid_index_with_updates.js
@@ -56,19 +56,19 @@ crudOpsForPhase(testDB.hybrid, 0);
 assert.eq(totalDocs, testDB.hybrid.count());
 
 // Hang the build after the first document.
-let stopKey = {'i': 1};
-turnFailPointOn("hangBeforeIndexBuildOf", stopKey);
+turnFailPointOn("hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", {fieldsToMatch: {i: 1}});
 
 // Start the background build.
 let bgBuild = startParallelShell(function() {
     assert.commandWorked(db.hybrid.createIndex({i: 1}, {background: true}));
 }, conn.port);
 
-if (isJsonLog(conn)) {
-    checkLog.contains(conn, /"id":20386,.*"attr":{"where":"before","i":1}/);
-} else {
-    checkLog.contains(conn, "Hanging before index build of i=1");
-}
+checkLog.containsJson(conn, 20386, {
+    where: "before",
+    doc: function(doc) {
+        return doc.i === 1;
+    }
+});
 
 // Phase 1: Collection scan and external sort
 // Insert documents while doing the bulk build.
@@ -79,7 +79,7 @@ assert.eq(totalDocs, testDB.hybrid.count());
 turnFailPointOn("hangAfterIndexBuildDumpsInsertsFromBulk");
 
 // Wait for the bulk insert to complete.
-turnFailPointOff("hangBeforeIndexBuildOf");
+turnFailPointOff("hangIndexBuildDuringCollectionScanPhaseBeforeInsertion");
 checkLog.contains(conn, "Hanging after dumping inserts from bulk builder");
 
 // Phase 2: First drain
diff --git a/jstests/noPassthrough/hybrid_unique_index_with_updates.js b/jstests/noPassthrough/hybrid_unique_index_with_updates.js
index 4fa394cd1cc..2b119996e52 100644
--- a/jstests/noPassthrough/hybrid_unique_index_with_updates.js
+++ b/jstests/noPassthrough/hybrid_unique_index_with_updates.js
@@ -20,18 +20,17 @@ let testDB = conn.getDB('test');
 // Enables a failpoint, runs 'hitFailpointFunc' to hit the failpoint, then runs
 // 'duringFailpointFunc' while the failpoint is active.
 let doDuringFailpoint = function(
-    failPointName, logMessage, structuredLogMessage, hitFailpointFunc, duringFailpointFunc, i) {
+    failPointName, structuredLogRegEx, hitFailpointFunc, duringFailpointFunc, stopKey) {
     clearRawMongoProgramOutput();
-    assert.commandWorked(
-        testDB.adminCommand({configureFailPoint: failPointName, mode: "alwaysOn", data: {"i": i}}));
+    assert.commandWorked(testDB.adminCommand({
+        configureFailPoint: failPointName,
+        mode: "alwaysOn",
+        data: {fieldsToMatch: {i: stopKey}}
+    }));
 
     hitFailpointFunc();
 
-    if (isJsonLogNoConn()) {
-        assert.soon(() => rawMongoProgramOutput().search(structuredLogMessage));
-    } else {
-        assert.soon(() => rawMongoProgramOutput().indexOf(logMessage) >= 0);
-    }
+    assert.soon(() => structuredLogRegEx.test(rawMongoProgramOutput()));
 
     duringFailpointFunc();
 
@@ -123,27 +122,26 @@ let runTest = function(config) {
             break;
         // Hang before scanning the first document.
         case 0:
-            doDuringFailpoint("hangBeforeIndexBuildOf",
-                              "Hanging before index build of i=" + stopKey,
-                              new RegExp("\"id\":20386.*\"where\":\"before\",\"i\":" + stopKey),
-                              buildIndex,
-                              doOperation,
-                              stopKey);
+            doDuringFailpoint(
+                "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion",
+                new RegExp("\"id\":20386.*\"where\":\"before\",\"doc\":.*\"i\":" + stopKey),
+                buildIndex,
+                doOperation,
+                stopKey);
             break;
         // Hang after scanning the first document.
         case 1:
-            doDuringFailpoint("hangAfterIndexBuildOf",
-                              "Hanging after index build of i=" + stopKey,
-                              new RegExp("\"id\":20386.*\"where\":\"after\",\"i\":" + stopKey),
-                              buildIndex,
-                              doOperation,
-                              stopKey);
+            doDuringFailpoint(
+                "hangIndexBuildDuringCollectionScanPhaseAfterInsertion",
+                new RegExp("\"id\":20386.*\"where\":\"after\",\"doc\":.*\"i\":" + stopKey),
+                buildIndex,
+                doOperation,
+                stopKey);
             break;
         // Hang before the first drain and after dumping the keys from the external sorter into
         // the index.
         case 2:
             doDuringFailpoint("hangAfterIndexBuildDumpsInsertsFromBulk",
-                              "Hanging after dumping inserts from bulk builder",
                               new RegExp("\"id\":20665"),
                               buildIndex,
                               doOperation);
@@ -151,7 +149,6 @@ let runTest = function(config) {
         // Hang before the second drain.
         case 3:
             doDuringFailpoint("hangAfterIndexBuildFirstDrain",
-                              "Hanging after index build first drain",
                               new RegExp("\"id\":20666"),
                               buildIndex,
                               doOperation);
@@ -159,7 +156,6 @@ let runTest = function(config) {
         // Hang before the final drain and commit.
         case 4:
             doDuringFailpoint("hangAfterIndexBuildSecondDrain",
-                              "Hanging after index build second drain",
                               new RegExp("\"id\":20667"),
                               buildIndex,
                               doOperation);
diff --git a/jstests/noPassthrough/indexbg2.js b/jstests/noPassthrough/indexbg2.js
index e4177e3922d..9410be45915 100644
--- a/jstests/noPassthrough/indexbg2.js
+++ b/jstests/noPassthrough/indexbg2.js
@@ -44,8 +44,8 @@ let waitParallel = function() {
 };
 
 let turnFailPointOn = function(failPointName, i) {
-    assert.commandWorked(
-        conn.adminCommand({configureFailPoint: failPointName, mode: "alwaysOn", data: {"i": i}}));
+    assert.commandWorked(conn.adminCommand(
+        {configureFailPoint: failPointName, mode: "alwaysOn", data: {fieldsToMatch: {i: i}}}));
 };
 
 let turnFailPointOff = function(failPointName) {
@@ -71,20 +71,24 @@ let failOnExistingDuplicateValue = function(coll) {
 let failOnInsertedDuplicateValue = function(coll) {
     let duplicateKey = 7;
 
-    turnFailPointOn("hangBeforeIndexBuildOf", duplicateKey);
+    turnFailPointOn("hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", duplicateKey);
 
     let bgIndexBuildPid;
     try {
         bgIndexBuildPid = indexBuild();
-        jsTestLog("Waiting to hang before index build of i=" + duplicateKey);
-        checkLog.contains(
-            conn,
-            new RegExp(`(Hanging before index build of i=${
-                duplicateKey}|Hanging.*index build of i.*"where":"before","i":${duplicateKey})`));
+
+        jsTestLog("Waiting to hang index build during collection scan before insertion of {i: " +
+                  duplicateKey + "}");
+        checkLog.containsJson(conn, 20386, {
+            where: "before",
+            doc: function(doc) {
+                return doc.i === duplicateKey;
+            }
+        });
 
         assert.commandWorked(coll.save({i: duplicateKey}));
     } finally {
-        turnFailPointOff("hangBeforeIndexBuildOf");
+        turnFailPointOff("hangIndexBuildDuringCollectionScanPhaseBeforeInsertion");
     }
 
     waitProgram(bgIndexBuildPid);
@@ -102,17 +106,20 @@ let failOnInsertedDuplicateValue = function(coll) {
 let succeedWithoutWriteErrors = function(coll, newKey) {
     let duplicateKey = 3;
 
-    turnFailPointOn("hangAfterIndexBuildOf", duplicateKey);
+    turnFailPointOn("hangIndexBuildDuringCollectionScanPhaseAfterInsertion", duplicateKey);
 
     let bgIndexBuildPid;
     try {
         bgIndexBuildPid = indexBuild();
 
-        jsTestLog("Waiting to hang after index build of i=" + duplicateKey);
-        checkLog.contains(
-            conn,
-            new RegExp(`(Hanging after index build of i=${
-                duplicateKey}|Hanging.*index build of i.*"where":"after","i":${duplicateKey})`));
+        jsTestLog("Waiting to hang index build during collection scan after insertion of {i: " +
+                  duplicateKey + "}");
+        checkLog.containsJson(conn, 20386, {
+            where: "after",
+            doc: function(doc) {
+                return doc.i === duplicateKey;
+            }
+        });
 
         assert.commandWorked(coll.insert({i: duplicateKey, n: true}));
 
@@ -124,7 +131,7 @@ let succeedWithoutWriteErrors = function(coll, newKey) {
         assert.commandWorked(coll.deleteOne({i: newKey, n: true}));
 
     } finally {
-        turnFailPointOff("hangAfterIndexBuildOf");
+        turnFailPointOff("hangIndexBuildDuringCollectionScanPhaseAfterInsertion");
     }
 
     waitProgram(bgIndexBuildPid);
diff --git a/jstests/noPassthrough/libs/index_build.js b/jstests/noPassthrough/libs/index_build.js
index 1befdfd1361..1e3c4044fe4 100644
--- a/jstests/noPassthrough/libs/index_build.js
+++ b/jstests/noPassthrough/libs/index_build.js
@@ -1,5 +1,9 @@
 // Helper functions for testing index builds.
 
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/parallel_shell_helpers.js");
+load("jstests/libs/uuid_util.js");
+
 var IndexBuildTest = class {
     /**
      * Starts an index build in a separate mongo shell process with given options.
@@ -207,7 +211,9 @@ var IndexBuildTest = class {
             .commandWorked(conn.adminCommand({getParameter: 1, enableIndexBuildCommitQuorum: 1}))
             .enableIndexBuildCommitQuorum;
     }
+};
 
+const ResumableIndexBuildTest = class {
     /**
      * Returns whether resumable index builds are supported.
      */
@@ -216,4 +222,92 @@ var IndexBuildTest = class {
             .commandWorked(conn.adminCommand({getParameter: 1, enableResumableIndexBuilds: 1}))
             .enableResumableIndexBuilds;
     }
+
+    /**
+     * Restarts the given node, ensuring that the the index build with name indexName has its state
+     * written to disk upon shutdown and is completed upon startup.
+     */
+    static restart(rst, conn, coll, indexName, failPointName) {
+        clearRawMongoProgramOutput();
+
+        const buildUUID = extractUUIDFromObject(
+            IndexBuildTest
+                .assertIndexes(coll, 2, ["_id_"], [indexName], {includeBuildUUIDs: true})[indexName]
+                .buildUUID);
+
+        const disableFailPoint = function(failPointName, buildUUID) {
+            // Wait for the log message that the index build has failed due to the node being shut
+            // down.
+            checkLog.containsJson(db.getMongo(), 20449, {
+                buildUUID: function(uuid) {
+                    return uuid["uuid"]["$uuid"] === buildUUID;
+                },
+                error: function(error) {
+                    return error.code === ErrorCodes.InterruptedDueToReplStateChange;
+                }
+            });
+
+            // Once the index build has failed, disable the failpoint so that shutdown can proceed.
+            assert.commandWorked(db.adminCommand({configureFailPoint: failPointName, mode: "off"}));
+        };
+        const awaitDisableFailPoint =
+            startParallelShell(funWithArgs(disableFailPoint, failPointName, buildUUID), conn.port);
+
+        rst.stop(conn);
+        awaitDisableFailPoint();
+
+        // Ensure that the resumable index build state was written to disk upon clean shutdown.
+        assert(RegExp("4841502.*" + buildUUID).test(rawMongoProgramOutput()));
+
+        rst.start(conn, {noCleanData: true});
+
+        // Ensure that the index build was completed upon the node starting back up.
+        checkLog.containsJson(conn, 20663, {
+            buildUUID: function(uuid) {
+                return uuid["uuid"]["$uuid"] === buildUUID;
+            },
+            namespace: coll.getFullName()
+        });
+        IndexBuildTest.assertIndexes(coll, 2, ["_id_", indexName]);
+    }
+
+    /**
+     * Runs the resumable index build test specified by the provided failpoint information and
+     * index spec on the provided replica set and namespace. Document(s) specified by
+     * insertIntoSideWritesTable will be inserted after the bulk load phase so that they are
+     * inserted into the side writes table and processed during the drain writes phase.
+     */
+    static run(
+        rst, dbName, collName, indexSpec, failPointName, failPointData, insertIntoSideWritesTable) {
+        const primary = rst.getPrimary();
+        const coll = primary.getDB(dbName).getCollection(collName);
+        const indexName = "resumable_index_build";
+
+        const fp = configureFailPoint(primary, failPointName, failPointData);
+
+        const createIndex = function(collName, indexSpec, indexName) {
+            assert.commandFailedWithCode(
+                db.getCollection(collName).createIndex(indexSpec, {name: indexName}),
+                ErrorCodes.InterruptedDueToReplStateChange);
+        };
+        const awaitCreateIndex = startParallelShell(
+            funWithArgs(createIndex, coll.getName(), indexSpec, indexName), primary.port);
+
+        if (insertIntoSideWritesTable) {
+            const sideWritesFp =
+                configureFailPoint(primary, "hangAfterIndexBuildDumpsInsertsFromBulk");
+            sideWritesFp.wait();
+
+            assert.commandWorked(coll.insert(insertIntoSideWritesTable));
+
+            sideWritesFp.off();
+        }
+
+        fp.wait();
+
+        ResumableIndexBuildTest.restart(rst, primary, coll, indexName, failPointName);
+
+        awaitCreateIndex();
+        assert.commandWorked(coll.dropIndex(indexName));
+    }
 };
diff --git a/jstests/noPassthrough/resumable_index_build_bulk_load_phase.js b/jstests/noPassthrough/resumable_index_build_bulk_load_phase.js
new file mode 100644
index 00000000000..cbf78d0ebc0
--- /dev/null
+++ b/jstests/noPassthrough/resumable_index_build_bulk_load_phase.js
@@ -0,0 +1,38 @@
+/**
+ * Tests that resumable index build state is written to disk upon clean shutdown when an index
+ * build is in the bulk load phase, and that the index build is subsequently completed when the
+ * node is started back up.
+ *
+ * @tags: [requires_persistence, requires_replication]
+ */
+(function() {
+"use strict";
+
+load("jstests/noPassthrough/libs/index_build.js");
+
+const dbName = "test";
+const collName = "resumable_index_build_bulk_load_phase";
+
+const rst = new ReplSetTest({nodes: 1});
+rst.startSet();
+rst.initiate();
+
+const primary = rst.getPrimary();
+const coll = primary.getDB(dbName).getCollection(collName);
+
+if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(primary)) {
+    jsTestLog("Skipping test because resumable index builds are not enabled");
+    rst.stopSet();
+    return;
+}
+
+assert.commandWorked(coll.insert({a: 1}));
+assert.commandWorked(coll.insert({a: 2}));
+
+ResumableIndexBuildTest.run(
+    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringBulkLoadPhase", {iteration: 0});
+ResumableIndexBuildTest.run(
+    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringBulkLoadPhase", {iteration: 1});
+
+rst.stopSet();
+})();
\ No newline at end of file
diff --git a/jstests/noPassthrough/resumable_index_build_collection_scan_phase.js b/jstests/noPassthrough/resumable_index_build_collection_scan_phase.js
new file mode 100644
index 00000000000..f4982f0ca08
--- /dev/null
+++ b/jstests/noPassthrough/resumable_index_build_collection_scan_phase.js
@@ -0,0 +1,42 @@
+/**
+ * Tests that resumable index build state is written to disk upon clean shutdown when an index
+ * build is in the collection scan phase, and that the index build is subsequently completed when
+ * the node is started back up.
+ *
+ * @tags: [requires_persistence, requires_replication]
+ */
+(function() {
+"use strict";
+
+load("jstests/noPassthrough/libs/index_build.js");
+
+const dbName = "test";
+const collName = "resumable_index_build_collection_scan_phase";
+
+const rst = new ReplSetTest({nodes: 1});
+rst.startSet();
+rst.initiate();
+
+const primary = rst.getPrimary();
+const coll = primary.getDB(dbName).getCollection(collName);
+
+if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(primary)) {
+    jsTestLog("Skipping test because resumable index builds are not enabled");
+    rst.stopSet();
+    return;
+}
+
+assert.commandWorked(coll.insert({a: 1}));
+assert.commandWorked(coll.insert({a: 2}));
+
+ResumableIndexBuildTest.run(
+    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", {
+        fieldsToMatch: {a: 1}
+    });
+ResumableIndexBuildTest.run(
+    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", {
+        fieldsToMatch: {a: 2}
+    });
+
+rst.stopSet();
+})();
\ No newline at end of file
diff --git a/jstests/noPassthrough/resumable_index_build_drain_writes_phase.js b/jstests/noPassthrough/resumable_index_build_drain_writes_phase.js
new file mode 100644
index 00000000000..6a97d680927
--- /dev/null
+++ b/jstests/noPassthrough/resumable_index_build_drain_writes_phase.js
@@ -0,0 +1,47 @@
+/**
+ * Tests that resumable index build state is written to disk upon clean shutdown when an index
+ * build is in the drain writes phase, and that the index build is subsequently completed when the
+ * node is started back up.
+ *
+ * @tags: [requires_persistence, requires_replication]
+ */
+(function() {
+"use strict";
+
+load("jstests/noPassthrough/libs/index_build.js");
+
+const dbName = "test";
+const collName = "resumable_index_build_drain_writes_phase";
+
+const rst = new ReplSetTest({nodes: 1});
+rst.startSet();
+rst.initiate();
+
+const primary = rst.getPrimary();
+const coll = primary.getDB(dbName).getCollection(collName);
+
+if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(primary)) {
+    jsTestLog("Skipping test because resumable index builds are not enabled");
+    rst.stopSet();
+    return;
+}
+
+assert.commandWorked(coll.insert({a: 1}));
+
+ResumableIndexBuildTest.run(rst,
+                            dbName,
+                            collName,
+                            {a: 1},
+                            "hangIndexBuildDuringDrainWritesPhase",
+                            {iteration: 0},
+                            [{a: 2}, {a: 3}]);
+ResumableIndexBuildTest.run(rst,
+                            dbName,
+                            collName,
+                            {a: 1},
+                            "hangIndexBuildDuringDrainWritesPhase",
+                            {iteration: 1},
+                            [{a: 4}, {a: 5}]);
+
+rst.stopSet();
+})();
\ No newline at end of file
diff --git a/jstests/replsets/hybrid_index_build_majority_commit_quorum_behavior.js b/jstests/replsets/hybrid_index_build_majority_commit_quorum_behavior.js
index bc69e47d7fb..50b4d35ea0b 100644
--- a/jstests/replsets/hybrid_index_build_majority_commit_quorum_behavior.js
+++ b/jstests/replsets/hybrid_index_build_majority_commit_quorum_behavior.js
@@ -7,7 +7,6 @@
 "use strict";
 load("jstests/replsets/rslib.js");
 load('jstests/noPassthrough/libs/index_build.js');
-load("jstests/libs/parallel_shell_helpers.js");  // funWithArgs
 
 var rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]});
 rst.startSet();
diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp
index 71db6045749..46d4604c113 100644
--- a/src/mongo/db/catalog/multi_index_block.cpp
+++ b/src/mongo/db/catalog/multi_index_block.cpp
@@ -36,6 +36,7 @@
 #include <ostream>
 
 #include "mongo/base/error_codes.h"
+#include "mongo/bson/simple_bsonelement_comparator.h"
 #include "mongo/db/audit.h"
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/collection_catalog.h"
@@ -66,8 +67,8 @@ MONGO_FAIL_POINT_DEFINE(hangAfterSettingUpIndexBuild);
 MONGO_FAIL_POINT_DEFINE(hangAfterSettingUpIndexBuildUnlocked);
 MONGO_FAIL_POINT_DEFINE(hangAfterStartingIndexBuild);
 MONGO_FAIL_POINT_DEFINE(hangAfterStartingIndexBuildUnlocked);
-MONGO_FAIL_POINT_DEFINE(hangBeforeIndexBuildOf);
-MONGO_FAIL_POINT_DEFINE(hangAfterIndexBuildOf);
+MONGO_FAIL_POINT_DEFINE(hangIndexBuildDuringCollectionScanPhaseBeforeInsertion);
+MONGO_FAIL_POINT_DEFINE(hangIndexBuildDuringCollectionScanPhaseAfterInsertion);
 MONGO_FAIL_POINT_DEFINE(leaveIndexBuildUnfinishedForShutdown);
 
 MultiIndexBlock::~MultiIndexBlock() {
@@ -310,14 +311,20 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(OperationContext* opCtx,
 void failPointHangDuringBuild(FailPoint* fp, StringData where, const BSONObj& doc) {
     fp->executeIf(
         [&](const BSONObj& data) {
-            int i = doc.getIntField("i");
-            LOGV2(
-                20386, "Hanging {where} index build of i={i}", "where"_attr = where, "i"_attr = i);
+            LOGV2(20386,
+                  "Hanging index build during collection scan phase insertion",
+                  "where"_attr = where,
+                  "doc"_attr = doc);
+
             fp->pauseWhileSet();
         },
-        [&](const BSONObj& data) {
-            int i = doc.getIntField("i");
-            return data["i"].numberInt() == i;
+        [&doc](const BSONObj& data) {
+            auto fieldsToMatch = data.getObjectField("fieldsToMatch");
+            return std::all_of(
+                fieldsToMatch.begin(), fieldsToMatch.end(), [&doc](const auto& elem) {
+                    return SimpleBSONElementComparator::kInstance.evaluate(elem ==
+                                                                           doc[elem.fieldName()]);
+                });
         });
 }
 
@@ -407,7 +414,8 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(OperationContext* opCtx,
 
             progress->setTotalWhileRunning(collection->numRecords(opCtx));
 
-            failPointHangDuringBuild(&hangBeforeIndexBuildOf, "before", objToIndex);
+            failPointHangDuringBuild(
+                &hangIndexBuildDuringCollectionScanPhaseBeforeInsertion, "before", objToIndex);
 
             // The external sorter is not part of the storage engine and therefore does not need a
             // WriteUnitOfWork to write keys.
@@ -416,7 +424,8 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(OperationContext* opCtx,
                 return ret;
             }
 
-            failPointHangDuringBuild(&hangAfterIndexBuildOf, "after", objToIndex);
+            failPointHangDuringBuild(
+                &hangIndexBuildDuringCollectionScanPhaseAfterInsertion, "after", objToIndex);
 
             // Go to the next document.
             progress->hit();
diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp
index 6f450b57767..b51b525f7aa 100644
--- a/src/mongo/db/index/index_access_method.cpp
+++ b/src/mongo/db/index/index_access_method.cpp
@@ -65,6 +65,8 @@ using std::set;
 
 using IndexVersion = IndexDescriptor::IndexVersion;
 
+MONGO_FAIL_POINT_DEFINE(hangIndexBuildDuringBulkLoadPhase);
+
 namespace {
 
 // Reserved RecordId against which multikey metadata keys are indexed.
@@ -638,7 +640,7 @@ Status AbstractIndexAccessMethod::commitBulk(OperationContext* opCtx,
 
     KeyString::Value previousKey;
 
-    while (it->more()) {
+    for (int64_t i = 0; it->more(); i++) {
         opCtx->checkForInterrupt();
 
         WriteUnitOfWork wunit(opCtx);
@@ -682,6 +684,17 @@ Status AbstractIndexAccessMethod::commitBulk(OperationContext* opCtx,
             }
         }
 
+        hangIndexBuildDuringBulkLoadPhase.executeIf(
+            [i](const BSONObj& data) {
+                LOGV2(4924400,
+                      "Hanging index build during bulk load phase due to "
+                      "'hangIndexBuildDuringBulkLoadPhase' failpoint",
+                      "iteration"_attr = i);
+
+                hangIndexBuildDuringBulkLoadPhase.pauseWhileSet();
+            },
+            [i](const BSONObj& data) { return i == data["iteration"].numberLong(); });
+
         Status status = builder->addKey(data.first);
 
         if (!status.isOK()) {
diff --git a/src/mongo/db/index/index_build_interceptor.cpp b/src/mongo/db/index/index_build_interceptor.cpp
index 8d319e51536..f3982216068 100644
--- a/src/mongo/db/index/index_build_interceptor.cpp
+++ b/src/mongo/db/index/index_build_interceptor.cpp
@@ -53,6 +53,7 @@
 namespace mongo {
 
 MONGO_FAIL_POINT_DEFINE(hangDuringIndexBuildDrainYield);
+MONGO_FAIL_POINT_DEFINE(hangIndexBuildDuringDrainWritesPhase);
 
 bool IndexBuildInterceptor::typeCanFastpathMultikeyUpdates(IndexType indexType) {
     // Ensure no new indexes are added without considering whether they use the multikeyPaths
@@ -186,6 +187,19 @@ Status IndexBuildInterceptor::drainWritesIntoIndex(OperationContext* opCtx,
                 break;
             }
 
+            hangIndexBuildDuringDrainWritesPhase.executeIf(
+                [&](const BSONObj& data) {
+                    LOGV2(4924401,
+                          "Hanging index build during drain writes phase due to "
+                          "'hangIndexBuildDuringDrainWritesPhase' failpoint",
+                          "iteration"_attr = _numApplied + batchSize);
+
+                    hangIndexBuildDuringDrainWritesPhase.pauseWhileSet();
+                },
+                [&](const BSONObj& data) {
+                    return _numApplied + batchSize == data["iteration"].numberLong();
+                });
+
             batchSize += 1;
             batchSizeBytes += objSize;