SERVER-44186 allow index builds to continue running on stepdown

This applies to two phase index builds only. Previously, the index build
would be aborted on stepdown.

5 files changed, 76 insertions, 15 deletions

diff --git a/jstests/noPassthrough/hybrid_index_build_on_state_transition.js b/jstests/noPassthrough/hybrid_index_build_on_state_transition.js
index fe093daa669..9bd185ac471 100644
--- a/jstests/noPassthrough/hybrid_index_build_on_state_transition.js
+++ b/jstests/noPassthrough/hybrid_index_build_on_state_transition.js
@@ -12,7 +12,14 @@
  * 5) 'abortTransaction' cmd attempts to acquire the RSTL lock in MODE_IX but blocked
  *    behind the step down thread.
  *
- * @tags: [uses_transactions, uses_prepare_transaction]
+ * TODO(SERVER-44190): Update this test to work with two phase index builds. Under two phase index
+ * builds, index builds are no longer aborted on stepdown.
+ *
+ * @tags: [
+ *     uses_transactions,
+ *     uses_prepare_transaction,
+ *     two_phase_index_builds_unsupported,
+ * ]
  */
 load("jstests/libs/check_log.js");
 load("jstests/replsets/rslib.js");
diff --git a/jstests/noPassthrough/index_stepdown_after_init.js b/jstests/noPassthrough/index_stepdown_after_init.js
index f52fb44899e..a20beef7f00 100644
--- a/jstests/noPassthrough/index_stepdown_after_init.js
+++ b/jstests/noPassthrough/index_stepdown_after_init.js
@@ -52,13 +52,28 @@ IndexBuildTest.waitForIndexBuildToStop(testDB);
 const exitCode = createIdx({checkExitSuccess: false});
 assert.neq(0, exitCode, 'expected shell to exit abnormally due to index build being terminated');
 
-// Wait for the IndexBuildCoordinator thread, not the command thread, to report the index build as
-// failed.
-checkLog.contains(primary, '[IndexBuildsCoordinatorMongod-0] Index build failed: ');
+const enableTwoPhaseIndexBuild =
+    assert.commandWorked(primary.adminCommand({getParameter: 1, enableTwoPhaseIndexBuild: 1}))
+        .enableTwoPhaseIndexBuild;
+if (!enableTwoPhaseIndexBuild) {
+    // Wait for the IndexBuildCoordinator thread, not the command thread, to report the index build
+    // as failed.
+    checkLog.contains(primary, '[IndexBuildsCoordinatorMongod-0] Index build failed: ');
 
-// Check that no new index has been created.  This verifies that the index build was aborted
-// rather than successfully completed.
-IndexBuildTest.assertIndexes(coll, 1, ['_id_']);
+    // Check that no new index has been created.  This verifies that the index build was aborted
+    // rather than successfully completed.
+    IndexBuildTest.assertIndexes(coll, 1, ['_id_']);
+    rst.stopSet();
+    return;
+}
+
+// With two phase index builds, a stepdown will not abort the index build, which should complete
+// after the node becomes primary again.
+rst.awaitReplication();
+IndexBuildTest.assertIndexes(coll, 2, ['_id_', 'a_1']);
+
+const secondaryColl = rst.getSecondary().getCollection(coll.getFullName());
+IndexBuildTest.assertIndexes(secondaryColl, 2, ['_id_', 'a_1']);
 
 rst.stopSet();
 })();
diff --git a/jstests/noPassthrough/index_stepdown_during_scan.js b/jstests/noPassthrough/index_stepdown_during_scan.js
index 9628318c208..0ddf2c378f7 100644
--- a/jstests/noPassthrough/index_stepdown_during_scan.js
+++ b/jstests/noPassthrough/index_stepdown_during_scan.js
@@ -51,9 +51,24 @@ IndexBuildTest.waitForIndexBuildToStop(testDB);
 const exitCode = createIdx({checkExitSuccess: false});
 assert.neq(0, exitCode, 'expected shell to exit abnormally due to index build being terminated');
 
-// Check that no new index has been created.  This verifies that the index build was aborted
-// rather than successfully completed.
-IndexBuildTest.assertIndexes(coll, 1, ['_id_']);
+const enableTwoPhaseIndexBuild =
+    assert.commandWorked(primary.adminCommand({getParameter: 1, enableTwoPhaseIndexBuild: 1}))
+        .enableTwoPhaseIndexBuild;
+if (!enableTwoPhaseIndexBuild) {
+    // Check that no new index has been created.  This verifies that the index build was aborted
+    // rather than successfully completed.
+    IndexBuildTest.assertIndexes(coll, 1, ['_id_']);
+    rst.stopSet();
+    return;
+}
+
+// With two phase index builds, a stepdown will not abort the index build, which should complete
+// after the node becomes primary again.
+rst.awaitReplication();
+IndexBuildTest.assertIndexes(coll, 2, ['_id_', 'a_1']);
+
+const secondaryColl = rst.getSecondary().getCollection(coll.getFullName());
+IndexBuildTest.assertIndexes(secondaryColl, 2, ['_id_', 'a_1']);
 
 rst.stopSet();
 })();
diff --git a/jstests/noPassthrough/index_stepdown_failover.js b/jstests/noPassthrough/index_stepdown_failover.js
index cf08ba71d16..09e1a809952 100644
--- a/jstests/noPassthrough/index_stepdown_failover.js
+++ b/jstests/noPassthrough/index_stepdown_failover.js
@@ -49,23 +49,31 @@ const newPrimaryDB = secondaryDB;
 const newPrimaryColl = secondaryColl;
 
 // Step down the primary.
-// Expect failed createIndex command invocation in parallel shell due to stepdown.
-// Before SERVER-44186, the index build will be aborted during stepdown.
+// Expect failed createIndex command invocation in parallel shell due to stepdown even though
+// the index build will continue in the background.
 assert.commandWorked(primary.adminCommand({replSetStepDown: 60, force: true}));
 const exitCode = createIdx({checkExitSuccess: false});
 assert.neq(0, exitCode, 'expected shell to exit abnormally due to index build being terminated');
 checkLog.contains(primary, 'Index build interrupted: ');
 
 // Unblock the index build on the old primary during the collection scanning phase.
+// This index build will not complete because it has to wait for a commitIndexBuild oplog
+// entry.
 IndexBuildTest.resumeIndexBuilds(primary);
+checkLog.contains(primary,
+                  'Index build waiting for commit or abort before completing final phase: ');
 
 // Step up the new primary.
 rst.stepUp(newPrimary);
 
-// A new index should not be present on the old primary because the index build was aborted.
+// A new index should be present on the old primary after processing the commitIndexBuild oplog
+// entry from the new primary.
 IndexBuildTest.waitForIndexBuildToStop(testDB);
-IndexBuildTest.assertIndexes(coll, 1, ['_id_']);
+IndexBuildTest.assertIndexes(coll, 2, ['_id_', 'a_1']);
+
+// Check that index was created on the new primary.
+IndexBuildTest.waitForIndexBuildToStop(newPrimaryDB);
+IndexBuildTest.assertIndexes(newPrimaryColl, 2, ['_id_', 'a_1']);
 
-TestData.skipCheckDBHashes = true;
 rst.stopSet();
 })();
diff --git a/src/mongo/db/commands/create_indexes.cpp b/src/mongo/db/commands/create_indexes.cpp
index 3567b3509ba..456649c6b14 100644
--- a/src/mongo/db/commands/create_indexes.cpp
+++ b/src/mongo/db/commands/create_indexes.cpp
@@ -731,6 +731,15 @@ bool runCreateIndexesWithCoordinator(OperationContext* opCtx,
         } catch (const ExceptionForCat<ErrorCategory::Interruption>& interruptionEx) {
             log() << "Index build interrupted: " << buildUUID << ": " << interruptionEx;
 
+            // If this node is no longer a primary, the index build will continue to run in the
+            // background and will complete when this node receives a commitIndexBuild oplog entry
+            // from the new primary.
+            if (indexBuildsCoord->supportsTwoPhaseIndexBuild() &&
+                ErrorCodes::InterruptedDueToReplStateChange == interruptionEx.code()) {
+                log() << "Index build continuing in background: " << buildUUID;
+                throw;
+            }
+
             // It is unclear whether the interruption originated from the current opCtx instance
             // for the createIndexes command or that the IndexBuildsCoordinator task was interrupted
             // independently of this command invocation. We'll defensively abort the index build
@@ -748,6 +757,13 @@ bool runCreateIndexesWithCoordinator(OperationContext* opCtx,
             log() << "Index build interrupted due to change in replication state: " << buildUUID
                   << ": " << ex;
 
+            // The index build will continue to run in the background and will complete when this
+            // node receives a commitIndexBuild oplog entry from the new primary.
+            if (indexBuildsCoord->supportsTwoPhaseIndexBuild()) {
+                log() << "Index build continuing in background: " << buildUUID;
+                throw;
+            }
+
             indexBuildsCoord->abortIndexBuildByBuildUUID(
                 opCtx,
                 buildUUID,