SERVER-49448 Interrupt index builds for shutdown during the expected phase in resumable index build tests

5 files changed, 90 insertions, 98 deletions

diff --git a/jstests/noPassthrough/libs/index_build.js b/jstests/noPassthrough/libs/index_build.js
index 3f6a19dfa37..b647e715c5d 100644
--- a/jstests/noPassthrough/libs/index_build.js
+++ b/jstests/noPassthrough/libs/index_build.js
@@ -239,29 +239,6 @@ const ResumableIndexBuildTest = class {
     }
 
     /**
-     * Waits for the specified index build to be interrupted and then disables the given failpoint.
-     */
-    static disableFailPointAfterInterruption(conn, failPointName, buildUUID) {
-        return startParallelShell(
-            funWithArgs(function(failPointName, buildUUID) {
-                // Wait for the index build to be interrupted.
-                checkLog.containsJson(db.getMongo(), 20449, {
-                    buildUUID: function(uuid) {
-                        return uuid["uuid"]["$uuid"] === buildUUID;
-                    },
-                    error: function(error) {
-                        return error.code === ErrorCodes.InterruptedDueToReplStateChange;
-                    }
-                });
-
-                // Once the index build has been interrupted, disable the failpoint so that shutdown
-                // or stepdown can proceed.
-                assert.commandWorked(
-                    db.adminCommand({configureFailPoint: failPointName, mode: "off"}));
-            }, failPointName, buildUUID), conn.port);
-    }
-
-    /**
      * Inserts the given documents once an index build reaches the end of the bulk load phase so
      * that the documents are inserted into the side writes table for that index build.
      */
@@ -299,7 +276,7 @@ const ResumableIndexBuildTest = class {
      * Restarts the given node, ensuring that the the index build with name indexName has its state
      * written to disk upon shutdown and is completed upon startup.
      */
-    static restart(rst, conn, coll, indexName, failPointName) {
+    static restart(rst, conn, coll, indexName, failPointName, failPointData) {
         clearRawMongoProgramOutput();
 
         const buildUUID = extractUUIDFromObject(
@@ -307,11 +284,42 @@ const ResumableIndexBuildTest = class {
                 .assertIndexes(coll, 2, ["_id_"], [indexName], {includeBuildUUIDs: true})[indexName]
                 .buildUUID);
 
-        const awaitDisableFailPoint = ResumableIndexBuildTest.disableFailPointAfterInterruption(
-            conn, failPointName, buildUUID);
+        // Don't interrupt the index build for shutdown until it is at the desired point.
+        const shutdownFpTimesEntered =
+            assert
+                .commandWorked(
+                    conn.adminCommand({configureFailPoint: "hangBeforeShutdown", mode: "alwaysOn"}))
+                .count;
+
+        const awaitContinueShutdown = startParallelShell(
+            funWithArgs(function(failPointName, failPointData, shutdownFpTimesEntered) {
+                load("jstests/libs/fail_point_util.js");
+
+                // Wait until we hang before shutdown to ensure that we do not move the index build
+                // forward before the step down process is complete.
+                assert.commandWorked(db.adminCommand({
+                    waitForFailPoint: "hangBeforeShutdown",
+                    timesEntered: shutdownFpTimesEntered + 1,
+                    maxTimeMS: kDefaultWaitForFailPointTimeout
+                }));
+
+                // Move the index build forward to the point that we want it to be interrupted for
+                // shutdown at.
+                const fp = configureFailPoint(db.getMongo(), failPointName, failPointData);
+                assert.commandWorked(db.adminCommand(
+                    {configureFailPoint: "hangAfterSettingUpIndexBuildUnlocked", mode: "off"}));
+                fp.wait();
+
+                // Disabling this failpoint will allow shutdown to continue and cause the operation
+                // context to be killed. This will cause the failpoint specified by failPointName
+                // to be interrupted and allow the index build to be interrupted for shutdown at
+                // its current location.
+                assert.commandWorked(
+                    db.adminCommand({configureFailPoint: "hangBeforeShutdown", mode: "off"}));
+            }, failPointName, failPointData, shutdownFpTimesEntered), conn.port);
 
         rst.stop(conn);
-        awaitDisableFailPoint();
+        awaitContinueShutdown();
 
         // Ensure that the resumable index build state was written to disk upon clean shutdown.
         assert(RegExp("4841502.*" + buildUUID).test(rawMongoProgramOutput()));
@@ -337,10 +345,16 @@ const ResumableIndexBuildTest = class {
                insertIntoSideWritesTable,
                postIndexBuildInserts = {}) {
         const primary = rst.getPrimary();
+
+        if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(primary)) {
+            jsTestLog("Skipping test because resumable index builds are not enabled");
+            return;
+        }
+
         const coll = primary.getDB(dbName).getCollection(collName);
         const indexName = "resumable_index_build";
 
-        const fp = configureFailPoint(primary, failPointName, failPointData);
+        const fp = configureFailPoint(primary, "hangAfterSettingUpIndexBuildUnlocked");
 
         const awaitInsertIntoSideWritesTable = ResumableIndexBuildTest.insertIntoSideWritesTable(
             primary, collName, insertIntoSideWritesTable);
@@ -350,7 +364,8 @@ const ResumableIndexBuildTest = class {
 
         fp.wait();
 
-        ResumableIndexBuildTest.restart(rst, primary, coll, indexName, failPointName);
+        ResumableIndexBuildTest.restart(
+            rst, primary, coll, indexName, failPointName, failPointData);
 
         awaitInsertIntoSideWritesTable();
         awaitCreateIndex();
diff --git a/jstests/noPassthrough/resumable_index_build_bulk_load_phase.js b/jstests/noPassthrough/resumable_index_build_bulk_load_phase.js
index cbf78d0ebc0..c5807105737 100644
--- a/jstests/noPassthrough/resumable_index_build_bulk_load_phase.js
+++ b/jstests/noPassthrough/resumable_index_build_bulk_load_phase.js
@@ -3,7 +3,10 @@
  * build is in the bulk load phase, and that the index build is subsequently completed when the
  * node is started back up.
  *
- * @tags: [requires_persistence, requires_replication]
+ * @tags: [
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
  */
 (function() {
 "use strict";
@@ -11,28 +14,17 @@
 load("jstests/noPassthrough/libs/index_build.js");
 
 const dbName = "test";
-const collName = "resumable_index_build_bulk_load_phase";
+const failPointName = "hangIndexBuildDuringBulkLoadPhase";
 
 const rst = new ReplSetTest({nodes: 1});
 rst.startSet();
 rst.initiate();
 
-const primary = rst.getPrimary();
-const coll = primary.getDB(dbName).getCollection(collName);
+const coll = rst.getPrimary().getDB(dbName).getCollection(jsTestName());
+assert.commandWorked(coll.insert([{a: 1}, {a: 2}]));
 
-if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(primary)) {
-    jsTestLog("Skipping test because resumable index builds are not enabled");
-    rst.stopSet();
-    return;
-}
-
-assert.commandWorked(coll.insert({a: 1}));
-assert.commandWorked(coll.insert({a: 2}));
-
-ResumableIndexBuildTest.run(
-    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringBulkLoadPhase", {iteration: 0});
-ResumableIndexBuildTest.run(
-    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringBulkLoadPhase", {iteration: 1});
+ResumableIndexBuildTest.run(rst, dbName, coll.getName(), {a: 1}, failPointName, {iteration: 0});
+ResumableIndexBuildTest.run(rst, dbName, coll.getName(), {a: 1}, failPointName, {iteration: 1});
 
 rst.stopSet();
 })();
\ No newline at end of file
diff --git a/jstests/noPassthrough/resumable_index_build_collection_scan_phase.js b/jstests/noPassthrough/resumable_index_build_collection_scan_phase.js
index f4982f0ca08..6aa18a03341 100644
--- a/jstests/noPassthrough/resumable_index_build_collection_scan_phase.js
+++ b/jstests/noPassthrough/resumable_index_build_collection_scan_phase.js
@@ -3,7 +3,10 @@
  * build is in the collection scan phase, and that the index build is subsequently completed when
  * the node is started back up.
  *
- * @tags: [requires_persistence, requires_replication]
+ * @tags: [
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
  */
 (function() {
 "use strict";
@@ -11,32 +14,17 @@
 load("jstests/noPassthrough/libs/index_build.js");
 
 const dbName = "test";
-const collName = "resumable_index_build_collection_scan_phase";
+const failPointName = "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion";
 
 const rst = new ReplSetTest({nodes: 1});
 rst.startSet();
 rst.initiate();
 
-const primary = rst.getPrimary();
-const coll = primary.getDB(dbName).getCollection(collName);
+const coll = rst.getPrimary().getDB(dbName).getCollection(jsTestName());
+assert.commandWorked(coll.insert([{a: 1}, {a: 2}]));
 
-if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(primary)) {
-    jsTestLog("Skipping test because resumable index builds are not enabled");
-    rst.stopSet();
-    return;
-}
-
-assert.commandWorked(coll.insert({a: 1}));
-assert.commandWorked(coll.insert({a: 2}));
-
-ResumableIndexBuildTest.run(
-    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", {
-        fieldsToMatch: {a: 1}
-    });
-ResumableIndexBuildTest.run(
-    rst, dbName, collName, {a: 1}, "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", {
-        fieldsToMatch: {a: 2}
-    });
+ResumableIndexBuildTest.run(rst, dbName, coll.getName(), {a: 1}, failPointName, {iteration: 0});
+ResumableIndexBuildTest.run(rst, dbName, coll.getName(), {a: 1}, failPointName, {iteration: 1});
 
 rst.stopSet();
 })();
\ No newline at end of file
diff --git a/jstests/noPassthrough/resumable_index_build_drain_writes_phase.js b/jstests/noPassthrough/resumable_index_build_drain_writes_phase.js
index c6f78592a0a..fc4f0cd776b 100644
--- a/jstests/noPassthrough/resumable_index_build_drain_writes_phase.js
+++ b/jstests/noPassthrough/resumable_index_build_drain_writes_phase.js
@@ -3,7 +3,10 @@
  * build is in the drain writes phase, and that the index build is subsequently completed when the
  * node is started back up.
  *
- * @tags: [requires_persistence, requires_replication]
+ * @tags: [
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
  */
 (function() {
 "use strict";
@@ -11,38 +14,22 @@
 load("jstests/noPassthrough/libs/index_build.js");
 
 const dbName = "test";
-const collName = "resumable_index_build_drain_writes_phase";
+const failPointName = "hangIndexBuildDuringDrainWritesPhase";
 
 const rst = new ReplSetTest({nodes: 1});
 rst.startSet();
 rst.initiate();
 
-const primary = rst.getPrimary();
-const coll = primary.getDB(dbName).getCollection(collName);
-
-if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(primary)) {
-    jsTestLog("Skipping test because resumable index builds are not enabled");
-    rst.stopSet();
-    return;
-}
-
+const coll = rst.getPrimary().getDB(dbName).getCollection(jsTestName());
 assert.commandWorked(coll.insert({a: 1}));
 
-ResumableIndexBuildTest.run(rst,
-                            dbName,
-                            collName,
-                            {a: 1},
-                            "hangIndexBuildDuringDrainWritesPhase",
-                            {iteration: 0},
-                            [{a: 2}, {a: 3}]);
-ResumableIndexBuildTest.run(rst,
-                            dbName,
-                            collName,
-                            {a: 1},
-                            "hangIndexBuildDuringDrainWritesPhase",
-                            {iteration: 1},
-                            [{a: 4}, {a: 5}],
-                            [{a: 6}, {a: 7}]);
+ResumableIndexBuildTest.run(
+    rst, dbName, coll.getName(), {a: 1}, failPointName, {iteration: 0}, [{a: 2}, {a: 3}]);
+ResumableIndexBuildTest.run(
+    rst, dbName, coll.getName(), {a: 1}, failPointName, {iteration: 1}, [{a: 4}, {a: 5}], [
+        {a: 6},
+        {a: 7}
+    ]);
 
 rst.stopSet();
 })();
\ No newline at end of file
diff --git a/jstests/replsets/libs/rollback_resumable_index_build.js b/jstests/replsets/libs/rollback_resumable_index_build.js
index 8e2fb92901c..206acab18f3 100644
--- a/jstests/replsets/libs/rollback_resumable_index_build.js
+++ b/jstests/replsets/libs/rollback_resumable_index_build.js
@@ -33,7 +33,7 @@ const RollbackResumableIndexBuildTest = class {
         const coll = originalPrimary.getDB(dbName).getCollection(collName);
         const indexName = "rollback_resumable_index_build";
 
-        let rollbackEndFp =
+        const rollbackEndFp =
             configureFailPoint(originalPrimary, rollbackEndFailPointName, rollbackEndFailPointData);
         const rollbackStartFp = configureFailPoint(
             originalPrimary, rollbackStartFailPointName, rollbackStartFailPointData);
@@ -60,13 +60,23 @@ const RollbackResumableIndexBuildTest = class {
 
         // Disable the failpoint in a parallel shell so that the primary can step down when the
         // rollback test is transitioning to sync source operations before rollback.
-        const awaitDisableFailPointAfterInterruption =
-            ResumableIndexBuildTest.disableFailPointAfterInterruption(
-                originalPrimary, rollbackEndFp.failPointName, buildUUID);
+        const awaitDisableFailPointAfterContinuingInBackground = startParallelShell(
+            funWithArgs(function(failPointName, buildUUID) {
+                // Wait for the index build to be continue in the background.
+                checkLog.containsJson(db.getMongo(), 20442, {
+                    buildUUID: function(uuid) {
+                        return uuid["uuid"]["$uuid"] === buildUUID;
+                    }
+                });
+
+                // Disable the failpoint so that stepdown can proceed.
+                assert.commandWorked(
+                    db.adminCommand({configureFailPoint: failPointName, mode: "off"}));
+            }, rollbackEndFp.failPointName, buildUUID), originalPrimary.port);
 
         rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
 
-        awaitDisableFailPointAfterInterruption();
+        awaitDisableFailPointAfterContinuingInBackground();
 
         // The index creation will report as having failed due to InterruptedDueToReplStateChange,
         // but it is still building in the background.