SERVER-44393 IndexBuildsCoordinator::_buildIndexTwoPhase() saves indexing error from first two phases on secondary and waits for abort

2 files changed, 39 insertions, 9 deletions

diff --git a/jstests/noPassthrough/index_secondary_wait_for_abort.js b/jstests/noPassthrough/index_secondary_wait_for_abort.js
index af33cd4d452..bcf7c393902 100644
--- a/jstests/noPassthrough/index_secondary_wait_for_abort.js
+++ b/jstests/noPassthrough/index_secondary_wait_for_abort.js
@@ -9,6 +9,7 @@
 (function() {
 "use strict";
 
+load('jstests/libs/check_log.js');
 load('jstests/noPassthrough/libs/index_build.js');
 
 const rst = new ReplSetTest({
@@ -60,16 +61,17 @@ assert.commandWorked(
 const createIdx = IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {'a.0': 1});
 
 const secondary = rst.getSecondary();
+const secondaryDB = secondary.getDB(testDB.getName());
 try {
-    // Wait for the index build to start on the primary. We cannot check the secondary because it
-    // may have crashed by then due to the invalid document.
+    // Wait for the index build to start on the primary.
     const opId = IndexBuildTest.waitForIndexBuildToStart(testDB, coll.getName(), 'a.0_1');
     IndexBuildTest.assertIndexBuildCurrentOpContents(testDB, opId);
 
-    const fassertProcessExitCode = _isWindows() ? MongoRunner.EXIT_ABRUPT : MongoRunner.EXIT_ABORT;
-    rst.stop(secondary, undefined, {allowedExitCode: fassertProcessExitCode});
-    assert(rawMongoProgramOutput().match('Fatal assertion 51101 Location16746: Index build: '),
-           'Index build should have aborted on error.');
+    // The index build on the secondary will fail on the invalid document but will wait for the
+    // abortIndexBuild oplog entry from the primary.
+    const secondaryOpId =
+        IndexBuildTest.waitForIndexBuildToStart(secondaryDB, coll.getName(), 'a.0_1');
+    IndexBuildTest.assertIndexBuildCurrentOpContents(secondaryDB, secondaryOpId);
 } finally {
     testDB.adminCommand({configureFailPoint: 'hangAfterSettingUpIndexBuild', mode: 'off'});
 }
@@ -80,13 +82,20 @@ IndexBuildTest.waitForIndexBuildToStop(testDB);
 const exitCode = createIdx({checkExitSuccess: false});
 assert.neq(0, exitCode, 'expected shell to exit abnormally due to index build failing');
 
+// Confirm that the index build on the secondary failed because of the invalid document.
+checkLog.contains(secondary, 'Location16746: Ambiguous field name found in array');
+
 // Check indexes on primary.
+rst.awaitReplication();
 IndexBuildTest.assertIndexes(coll, 1, ['_id_']);
 
+// Check that index was not created on the secondary.
+const secondaryColl = secondaryDB.getCollection(coll.getName());
+IndexBuildTest.assertIndexes(secondaryColl, 1, ['_id_']);
+
 const cmdNs = testDB.getCollection('$cmd').getFullName();
 const ops = rst.dumpOplog(primary, {op: 'c', ns: cmdNs, 'o.abortIndexBuild': coll.getName()});
 assert.eq(1, ops.length, 'primary did not write abortIndexBuild oplog entry: ' + tojson(ops));
 
-TestData.skipCheckDBHashes = true;
 rst.stopSet();
 })();
diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp
index 102a431fc2c..9e9977f772b 100644
--- a/src/mongo/db/index_builds_coordinator.cpp
+++ b/src/mongo/db/index_builds_coordinator.cpp
@@ -1278,8 +1278,29 @@ void IndexBuildsCoordinator::_buildIndexTwoPhase(
     std::shared_ptr<ReplIndexBuildState> replState,
     const IndexBuildOptions& indexBuildOptions,
     boost::optional<Lock::CollectionLock>* exclusiveCollectionLock) {
-    _scanCollectionAndInsertKeysIntoSorter(opCtx, dbAndUUID, replState, exclusiveCollectionLock);
-    auto nss = _insertKeysFromSideTablesWithoutBlockingWrites(opCtx, dbAndUUID, replState);
+
+    auto nss = *CollectionCatalog::get(opCtx).lookupNSSByUUID(replState->collectionUUID);
+    auto preAbortStatus = Status::OK();
+    try {
+        _scanCollectionAndInsertKeysIntoSorter(
+            opCtx, dbAndUUID, replState, exclusiveCollectionLock);
+        nss = _insertKeysFromSideTablesWithoutBlockingWrites(opCtx, dbAndUUID, replState);
+    } catch (DBException& ex) {
+        auto replCoord = repl::ReplicationCoordinator::get(opCtx);
+        auto replSetAndNotPrimary =
+            replCoord->getSettings().usingReplSets() && !replCoord->canAcceptWritesFor(opCtx, nss);
+        if (!replSetAndNotPrimary) {
+            throw;
+        }
+        if (ErrorCodes::InterruptedAtShutdown == ex.code()) {
+            throw;
+        }
+        log() << "Index build failed before final phase during oplog application. "
+                 "Waiting for abort: "
+              << replState->buildUUID << ": " << ex;
+        preAbortStatus = ex.toStatus();
+    }
+
     auto commitIndexBuildTimestamp = _waitForCommitOrAbort(opCtx, nss, replState);
     _insertKeysFromSideTablesAndCommit(opCtx,
                                        dbAndUUID,