SERVER-47095 Secondaries voting to commit index builds already existing should handle exceptions

2 files changed, 153 insertions, 2 deletions

diff --git a/jstests/noPassthrough/rolling_index_builds_shutdown_early.js b/jstests/noPassthrough/rolling_index_builds_shutdown_early.js
new file mode 100644
index 00000000000..1f76c63c3f8
--- /dev/null
+++ b/jstests/noPassthrough/rolling_index_builds_shutdown_early.js
@@ -0,0 +1,129 @@
+/**
+ * Tests that secondaries already containing an index build started by the primary node vote for
+ * committing the index immediately and do not raise an exception if a shutdown occurs before the
+ * verdict.
+ *
+ * @tags: [
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
+ */
+
+(function() {
+'use strict';
+
+load('jstests/noPassthrough/libs/index_build.js');
+
+const replTest = new ReplSetTest({
+    nodes: [
+        {},
+        {
+            // Disallow elections on secondary.
+            rsConfig: {
+                priority: 0,
+                votes: 0,
+            }
+        },
+        {
+            // Disallow elections on secondary.
+            rsConfig: {
+                priority: 0,
+                votes: 0,
+            }
+        },
+    ]
+});
+
+replTest.startSet();
+replTest.initiate();
+
+const dbName = 'test';
+const collName = 't';
+
+TestData.dbName = dbName;
+TestData.collName = collName;
+
+let primary = replTest.getPrimary();
+let primaryDB = primary.getDB(dbName);
+let coll = primaryDB.getCollection(collName);
+
+if (!IndexBuildTest.supportsTwoPhaseIndexBuild(primary)) {
+    jsTestLog('Two phase index builds not supported, skipping test.');
+    replTest.stopSet();
+    return;
+}
+
+// Populate the collection to avoid empty collection index build optimization.
+assert.commandWorked(coll.insert({x: 1, y: 1}));
+
+// Make sure the documents make it to the secondaries.
+replTest.awaitReplication();
+
+let secondaries = replTest.getSecondaries();
+const standalonePort = allocatePort();
+jsTestLog('Standalone server will listen on port: ' + standalonePort);
+
+function buildIndexOnNodeAsStandalone(node) {
+    jsTestLog('A. Restarting as standalone: ' + node.host);
+    replTest.stop(node, /*signal=*/null, /*opts=*/null, {forRestart: true, waitpid: true});
+    const standalone = MongoRunner.runMongod({
+        restart: true,
+        dbpath: node.dbpath,
+        port: standalonePort,
+        setParameter: {
+            disableLogicalSessionCacheRefresh: true,
+            ttlMonitorEnabled: false,
+        },
+    });
+    if (jsTestOptions().keyFile) {
+        assert(jsTest.authenticate(standalone),
+               'Failed authentication during restart: ' + standalone.host);
+    }
+
+    jsTestLog('B. Building index on standalone: ' + standalone.host);
+    const standaloneDB = standalone.getDB(dbName);
+    const standaloneColl = standaloneDB.getCollection(collName);
+    assert.commandWorked(standaloneColl.createIndex({x: 1}, {name: 'rolling_index_x_1'}));
+
+    jsTestLog('C. Restarting as replica set node: ' + node.host);
+    MongoRunner.stopMongod(standalone);
+    replTest.restart(node);
+    replTest.awaitReplication();
+
+    let mongo = new Mongo(node.host);
+    mongo.setSlaveOk(true);
+    let indexes = mongo.getDB(dbName).getCollection(collName).getIndexes();
+    assert.eq(2, indexes.length);
+}
+
+buildIndexOnNodeAsStandalone(secondaries[0]);
+
+jsTestLog('D. Repeat the procedure for the remaining secondary: ' + secondaries[1].host);
+buildIndexOnNodeAsStandalone(secondaries[1]);
+
+replTest.awaitNodesAgreeOnPrimary(
+    replTest.kDefaultTimeoutMS, replTest.nodes, replTest.getNodeId(primary));
+
+secondaries = replTest.getSecondaries();
+
+// Enable fail points on the secondaries to prevent them from sending the commit quorum vote to the
+// primary.
+assert.commandWorked(secondaries[0].adminCommand(
+    {configureFailPoint: 'hangBeforeSendingCommitQuorumVote', mode: 'alwaysOn'}));
+assert.commandWorked(secondaries[1].adminCommand(
+    {configureFailPoint: 'hangBeforeSendingCommitQuorumVote', mode: 'alwaysOn'}));
+
+jsTestLog('E. Build index on the primary: ' + primary.host);
+startParallelShell(() => {
+    const coll = db.getSiblingDB(TestData.dbName).getCollection(TestData.collName);
+    coll.createIndex({x: 1}, {name: 'rolling_index_x_1'}, 3);
+}, primary.port);
+
+checkLog.containsJson(secondaries[0], 4709501);
+checkLog.containsJson(secondaries[1], 4709501);
+
+replTest.stop(secondaries[0], /*signal=*/null, /*opts=*/null, {waitpid: true});
+replTest.stop(secondaries[1], /*signal=*/null, /*opts=*/null, {waitpid: true});
+
+replTest.stop(primary, /*signal=*/null, /*opts=*/null, {waitpid: true});
+}());
diff --git a/src/mongo/db/index_builds_coordinator_mongod.cpp b/src/mongo/db/index_builds_coordinator_mongod.cpp
index 03427802f4a..e984cf2363a 100644
--- a/src/mongo/db/index_builds_coordinator_mongod.cpp
+++ b/src/mongo/db/index_builds_coordinator_mongod.cpp
@@ -58,6 +58,7 @@ using namespace indexbuildentryhelpers;
 namespace {
 
 MONGO_FAIL_POINT_DEFINE(hangBeforeInitializingIndexBuild);
+MONGO_FAIL_POINT_DEFINE(hangBeforeSendingCommitQuorumVote);
 MONGO_FAIL_POINT_DEFINE(hangAfterInitializingIndexBuild);
 
 /**
@@ -236,9 +237,25 @@ IndexBuildsCoordinatorMongod::startIndexBuild(OperationContext* opCtx,
             // Signal that the index build started successfully.
             startPromise.setWith([] {});
 
-            _signalPrimaryForCommitReadiness(opCtx.get(), replState);
-            _waitForNextIndexBuildAction(opCtx.get(), replState);
+            try {
+                _signalPrimaryForCommitReadiness(opCtx.get(), replState);
+                _waitForNextIndexBuildAction(opCtx.get(), replState);
+            } catch (const ExceptionForCat<ErrorCategory::Interruption>& ex) {
+                // This is a blocking call that can take an extended amount of time to finish. We
+                // need to anticipate possible shutdowns or interruptions here.
+                status = ex.toStatus();
+
+                LOGV2(4709502,
+                      "Vote interrupted for committing the index build",
+                      "collectionUUID"_attr = replState->collectionUUID,
+                      "buildUUID"_attr = replState->buildUUID,
+                      "indexNames"_attr = replState->indexNames,
+                      "indexSpecs"_attr = replState->indexSpecs,
+                      "status"_attr = status);
+            }
 
+            // No additional cleanup is necessary other than unregistering the index build as it's
+            // already built.
             {
                 stdx::unique_lock<Latch> lk(_mutex);
                 _unregisterIndexBuild(lk, replState);
@@ -532,6 +549,11 @@ void IndexBuildsCoordinatorMongod::_signalPrimaryForCommitReadiness(
 
         BSONObj voteCmdResponse;
         try {
+            if (MONGO_unlikely(hangBeforeSendingCommitQuorumVote.shouldFail())) {
+                LOGV2(4709501, "Hanging on 'hangBeforeSendingCommitQuorumVote' fail point.");
+                hangBeforeSendingCommitQuorumVote.pauseWhileSet(opCtx);
+            }
+
             voteCmdResponse = replCoord->runCmdOnPrimaryAndAwaitResponse(
                 opCtx, "admin", voteCmdRequest, onRemoteCmdScheduled, onRemoteCmdComplete);
         } catch (DBException& ex) {