SERVER-39562 Repair should handle duplicate unique index keys

Removed MongoDFixture from disk_wiredtiger suite.

10 files changed, 429 insertions, 82 deletions

diff --git a/buildscripts/resmokeconfig/suites/disk_wiredtiger.yml b/buildscripts/resmokeconfig/suites/disk_wiredtiger.yml
index f80b49a5383..4319ee4c2df 100644
--- a/buildscripts/resmokeconfig/suites/disk_wiredtiger.yml
+++ b/buildscripts/resmokeconfig/suites/disk_wiredtiger.yml
@@ -5,22 +5,10 @@ selector:
   - jstests/disk/*.js
 
 executor:
-  archive:
-    hooks:
-      - ValidateCollections
   config:
     shell_options:
+      nodb: ''
       global_vars:
         TestData:
           storageEngine: wiredTiger
       readMode: commands
-  hooks:
-  - class: ValidateCollections
-  - class: CleanEveryN
-    n: 1
-  fixture:
-    class: MongoDFixture
-    mongod_options:
-      set_parameters:
-        enableTestCommands: 1
-      storageEngine: wiredTiger
diff --git a/jstests/disk/directoryperdb.js b/jstests/disk/directoryperdb.js
index de2b2a272dd..58940a30ff4 100644
--- a/jstests/disk/directoryperdb.js
+++ b/jstests/disk/directoryperdb.js
@@ -1,16 +1,10 @@
 var baseDir = 'jstests_disk_directoryper';
 var baseName = 'directoryperdb';
 var dbpath = MongoRunner.dataPath + baseDir + '/';
-var storageEngine = db.serverStatus().storageEngine.name;
-
-// The pattern which matches the names of database files
-var dbFileMatcher;
-if (storageEngine == 'wiredTiger') {
-    // Matches wiredTiger collection-*.wt and index-*.wt files
-    dbFileMatcher = /(collection|index)-.+\.wt$/;
-} else {
-    assert(false, 'This test must be run against wiredTiger');
-}
+var storageEngine = "wiredTiger";
+
+// Matches wiredTiger collection-*.wt and index-*.wt files
+var dbFileMatcher = /(collection|index)-.+\.wt$/;
 
 // Set up helper functions.
 assertDocumentCount = function(db, count) {
@@ -24,22 +18,19 @@ assertDocumentCount = function(db, count) {
  * Returns the current connection which gets restarted with wiredtiger.
  */
 checkDBFilesInDBDirectory = function(conn, dbToCheck) {
-    if (storageEngine == 'wiredTiger') {
-        MongoRunner.stopMongod(conn);
-        conn = MongoRunner.runMongod({dbpath: dbpath, directoryperdb: '', restart: true});
-    }
+    MongoRunner.stopMongod(conn);
+    conn = MongoRunner.runMongod({dbpath: dbpath, directoryperdb: '', restart: true});
 
     var dir = dbpath + dbToCheck;
-    if (storageEngine == 'wiredTiger') {
-        // The KV catalog escapes non alpha-numeric characters with its UTF-8 byte sequence in
-        // decimal when creating the directory on disk.
-        if (dbToCheck == '&') {
-            dir = dbpath + '.38';
-        } else if (dbToCheck == '処') {
-            dir = dbpath + '.229.135.166';
-        } else if (dbToCheck == Array(22).join('処')) {
-            dir = dbpath + Array(22).join('.229.135.166');
-        }
+
+    // The KV catalog escapes non alpha-numeric characters with its UTF-8 byte sequence in
+    // decimal when creating the directory on disk.
+    if (dbToCheck == '&') {
+        dir = dbpath + '.38';
+    } else if (dbToCheck == '処') {
+        dir = dbpath + '.229.135.166';
+    } else if (dbToCheck == Array(22).join('処')) {
+        dir = dbpath + Array(22).join('.229.135.166');
     }
 
     files = listFiles(dir);
@@ -59,10 +50,8 @@ checkDBFilesInDBDirectory = function(conn, dbToCheck) {
  * Returns the restarted connection with wiredtiger.
  */
 checkDBDirectoryNonexistent = function(conn, dbToCheck) {
-    if (storageEngine == 'wiredTiger') {
-        MongoRunner.stopMongod(conn);
-        conn = MongoRunner.runMongod({dbpath: dbpath, directoryperdb: '', restart: true});
-    }
+    MongoRunner.stopMongod(conn);
+    conn = MongoRunner.runMongod({dbpath: dbpath, directoryperdb: '', restart: true});
 
     var files = listFiles(dbpath);
     // Check that there are no files in the toplevel dbpath.
@@ -75,20 +64,18 @@ checkDBDirectoryNonexistent = function(conn, dbToCheck) {
     }
 
     // Check db directories to ensure db files in them have been destroyed.
-    if (storageEngine == 'wiredTiger') {
-        var dir = dbpath + dbToCheck;
-        // The KV catalog escapes non alpha-numeric characters with its UTF-8 byte sequence in
-        // decimal when creating the directory on disk.
-        if (dbToCheck == '&') {
-            dir = dbpath + '.38';
-        } else if (dbToCheck == '処') {
-            dir = dbpath + '.229.135.166';
-        } else if (dbToCheck == Array(22).join('処')) {
-            dir = dbpath + Array(22).join('.229.135.166');
-        }
-        var files = listFiles(dir);
-        assert.eq(files.length, 0, 'Files left behind in database directory');
+    var dir = dbpath + dbToCheck;
+    // The KV catalog escapes non alpha-numeric characters with its UTF-8 byte sequence in
+    // decimal when creating the directory on disk.
+    if (dbToCheck == '&') {
+        dir = dbpath + '.38';
+    } else if (dbToCheck == '処') {
+        dir = dbpath + '.229.135.166';
+    } else if (dbToCheck == Array(22).join('処')) {
+        dir = dbpath + Array(22).join('.229.135.166');
     }
+    var files = listFiles(dir);
+    assert.eq(files.length, 0, 'Files left behind in database directory');
     return conn;
 };
 
@@ -119,10 +106,8 @@ assert.writeError(db[ 'journal' ].insert( {} ));
 
 // Using WiredTiger, it should be impossible to create a database named 'WiredTiger' with
 // directoryperdb, as that file is created by the WiredTiger storageEngine.
-if (storageEngine == 'wiredTiger') {
-    var dbW = m.getDB('WiredTiger');
-    assert.writeError(dbW[baseName].insert({}));
-}
+var dbW = m.getDB('WiredTiger');
+assert.writeError(dbW[baseName].insert({}));
 
 // Create a database named 'a' repeated 63 times.
 var dbNameAA = Array(64).join('a');
diff --git a/jstests/disk/repair_duplicate_keys.js b/jstests/disk/repair_duplicate_keys.js
new file mode 100644
index 00000000000..e761136ac24
--- /dev/null
+++ b/jstests/disk/repair_duplicate_keys.js
@@ -0,0 +1,259 @@
+/**
+ * Tests that --repair deletes documents containing duplicate unique keys and inserts them into a
+ * local lost and found collection.
+ *
+ * @tags: [requires_wiredtiger]
+ */
+
+(function() {
+
+load('jstests/disk/libs/wt_file_helper.js');
+load("jstests/libs/uuid_util.js");
+
+const baseName = "repair_duplicate_keys";
+const localBaseName = "local";
+const collName = "test";
+const lostAndFoundCollBaseName = "system.lost_and_found.";
+
+const dbpath = MongoRunner.dataPath + baseName + "/";
+const indexName = "a_1";
+const doc1 = {
+    a: 1,
+};
+const doc2 = {
+    a: 10,
+};
+const doc3 = {
+    a: 100,
+};
+const docWithId = {
+    a: 1,
+    _id: 1
+};
+const dupDocWithId = {
+    a: 1,
+    _id: 1
+};
+
+resetDbpath(dbpath);
+let port;
+
+// Initializes test collection for tests 1-3.
+let createIndexedCollWithDocs = function(coll) {
+    assert.commandWorked(coll.insert(doc1));
+    assert.commandWorked(coll.createIndex({a: 1}, {name: indexName, unique: true}));
+    assert.commandWorked(coll.insert(doc2));
+    assert.commandWorked(coll.insert(doc3));
+
+    assertQueryUsesIndex(coll, doc1, indexName);
+    assertQueryUsesIndex(coll, doc2, indexName);
+    assertQueryUsesIndex(coll, doc3, indexName);
+    return coll;
+};
+
+// Bypasses DuplicateKey insertion error for testing via failpoint.
+let addDuplicateDocumentToCol = function(db, coll, doc) {
+    jsTestLog("Insert document without index entries.");
+    assert.commandWorked(
+        db.adminCommand({configureFailPoint: "skipIndexNewRecords", mode: "alwaysOn"}));
+
+    assert.commandWorked(coll.insert(doc));
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "skipIndexNewRecords", mode: "off"}));
+};
+
+// Runs repair on collection with possible duplicate keys and verifies original documents in
+// collection initialized with "createIndexedCollWithDocs" are still present.
+let runRepairAndVerifyCollectionDocs = function() {
+    jsTestLog("Entering runRepairAndVerifyCollectionDocs...");
+
+    assertRepairSucceeds(dbpath, port);
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    let testColl = mongod.getDB(baseName)[collName];
+
+    assert.eq(testColl.find(doc1).itcount(), 1);
+    assert.eq(testColl.find(doc2).itcount(), 1);
+    assert.eq(testColl.find(doc3).itcount(), 1);
+    assert.eq(testColl.count(), 3);
+
+    MongoRunner.stopMongod(mongod);
+    jsTestLog("Exiting runRepairAndVerifyCollectionDocs.");
+};
+
+/* Test 1: Insert unique documents and verify that no local database is generated. */
+(function startStandaloneWithNoDup() {
+    jsTestLog("Entering startStandaloneWithNoDup...");
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    port = mongod.port;
+    let testColl = mongod.getDB(baseName)[collName];
+
+    testColl = createIndexedCollWithDocs(testColl);
+    assert.commandFailedWithCode(testColl.insert(doc1), [ErrorCodes.DuplicateKey]);
+    assert.commandFailedWithCode(testColl.insert(doc2), [ErrorCodes.DuplicateKey]);
+    assert.commandFailedWithCode(testColl.insert(doc3), [ErrorCodes.DuplicateKey]);
+
+    assert.eq(testColl.count(), 3);
+
+    MongoRunner.stopMongod(mongod);
+    jsTestLog("Exiting startStandaloneWithNoDup.");
+})();
+
+runRepairAndVerifyCollectionDocs();
+
+(function checkLostAndFoundCollForNoDup() {
+    jsTestLog("Entering checkLostAndFoundCollForNoDup...");
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    const uuid_obj = getUUIDFromListCollections(mongod.getDB(baseName), collName);
+    const uuid = extractUUIDFromObject(uuid_obj);
+
+    let localColl = mongod.getDB(localBaseName)[lostAndFoundCollBaseName + uuid];
+    assert.isnull(localColl.exists());
+
+    MongoRunner.stopMongod(mongod);
+    jsTestLog("Exiting checkLostAndFoundCollForNoDup.");
+})();
+
+/* Test 2: Insert one duplicate document into test collection and verify that repair deletes the
+ * document from the collection, generates a "local.lost_and_found" collection and inserts
+ * duplicate document into it. */
+(function startStandaloneWithOneDup() {
+    jsTestLog("Entering startStandaloneWithOneDup...");
+    resetDbpath(dbpath);
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    port = mongod.port;
+    let db = mongod.getDB(baseName);
+    let testColl = mongod.getDB(baseName)[collName];
+
+    testColl = createIndexedCollWithDocs(testColl);
+    assert.commandFailedWithCode(testColl.insert(doc1), [ErrorCodes.DuplicateKey]);
+
+    addDuplicateDocumentToCol(db, testColl, doc1);
+
+    assert.eq(testColl.count(), 4);
+
+    MongoRunner.stopMongod(mongod, undefined, {skipValidation: true});
+    jsTestLog("Exiting startStandaloneWithOneDup.");
+})();
+
+runRepairAndVerifyCollectionDocs();
+
+(function checkLostAndFoundCollForOneDup() {
+    jsTestLog("Entering checkLostAndFoundCollForOneDup...");
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    const uuid_obj = getUUIDFromListCollections(mongod.getDB(baseName), collName);
+    const uuid = extractUUIDFromObject(uuid_obj);
+
+    let localColl = mongod.getDB(localBaseName)[lostAndFoundCollBaseName + uuid];
+    assert.eq(localColl.find(doc1).itcount(), 1);
+    assert.eq(localColl.count(), 1);
+
+    MongoRunner.stopMongod(mongod);
+    jsTestLog("Exiting checkLostAndFoundCollForOneDup.");
+})();
+
+/* Test 3: Insert multiple duplicate documents into the test collection and verify that repair
+ * deletes the documents from the collection, generates a "local.lost_and_found" collection and
+ * inserts duplicate document into it.
+ */
+(function startStandaloneWithMultipleDups() {
+    jsTestLog("Entering startStandaloneWithMultipleDups...");
+    resetDbpath(dbpath);
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    port = mongod.port;
+    let db = mongod.getDB(baseName);
+    let testColl = mongod.getDB(baseName)[collName];
+
+    testColl = createIndexedCollWithDocs(testColl);
+    assert.commandFailedWithCode(testColl.insert(doc1), [ErrorCodes.DuplicateKey]);
+
+    addDuplicateDocumentToCol(db, testColl, doc1);
+    addDuplicateDocumentToCol(db, testColl, doc2);
+    addDuplicateDocumentToCol(db, testColl, doc3);
+
+    assert.eq(testColl.count(), 6);
+
+    MongoRunner.stopMongod(mongod, undefined, {skipValidation: true});
+    jsTestLog("Exiting startStandaloneWithMultipleDups.");
+})();
+
+runRepairAndVerifyCollectionDocs();
+
+(function checkLostAndFoundCollForDups() {
+    jsTestLog("Entering checkLostAndFoundCollForDups...");
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    const uuid_obj = getUUIDFromListCollections(mongod.getDB(baseName), collName);
+    const uuid = extractUUIDFromObject(uuid_obj);
+
+    let localColl = mongod.getDB(localBaseName)[lostAndFoundCollBaseName + uuid];
+
+    assert.eq(localColl.find(doc1).itcount(), 1);
+    assert.eq(localColl.find(doc2).itcount(), 1);
+    assert.eq(localColl.find(doc3).itcount(), 1);
+    assert.eq(localColl.count(), 3);
+
+    MongoRunner.stopMongod(mongod);
+    jsTestLog("Exiting checkLostAndFoundCollForDups.");
+})();
+
+/* Test 4: Insert document with both duplicate "a" field and duplicate _id field. Verify that repair
+ * deletes the duplicate document from test collection, generates a "local.lost_and_found"
+ * collection and inserts duplicate document into it.
+ */
+(function startStandaloneWithDoubleDup() {
+    jsTestLog("Entering startStandaloneWithDoubleDup...");
+    resetDbpath(dbpath);
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    port = mongod.port;
+    let db = mongod.getDB(baseName);
+    let testColl = mongod.getDB(baseName)[collName];
+
+    assert.commandWorked(testColl.insert(docWithId));
+    assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName, unique: true}));
+
+    addDuplicateDocumentToCol(db, testColl, dupDocWithId);
+
+    assert.eq(testColl.count(), 2);
+
+    MongoRunner.stopMongod(mongod, undefined, {skipValidation: true});
+    jsTestLog("Exiting startStandaloneWithDoubleDup.");
+})();
+
+(function runRepairAndVerifyCollectionDoc() {
+    jsTestLog("Running repair...");
+
+    assertRepairSucceeds(dbpath, port);
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    let testColl = mongod.getDB(baseName)[collName];
+
+    assert.eq(testColl.find(docWithId).itcount(), 1);
+    assert.eq(testColl.count(), 1);
+
+    MongoRunner.stopMongod(mongod);
+
+    jsTestLog("Finished repairing.");
+})();
+
+(function checkLostAndFoundCollForDoubleDup() {
+    jsTestLog("Entering checkLostAndFoundCollForDoubleDup...");
+
+    let mongod = startMongodOnExistingPath(dbpath);
+    const uuid_obj = getUUIDFromListCollections(mongod.getDB(baseName), collName);
+    const uuid = extractUUIDFromObject(uuid_obj);
+
+    let localColl = mongod.getDB(localBaseName)[lostAndFoundCollBaseName + uuid];
+    assert.eq(localColl.find(docWithId).itcount(), 1);
+    assert.eq(localColl.count(), 1);
+
+    MongoRunner.stopMongod(mongod);
+    jsTestLog("Exiting checkLostAndFoundCollForDoubleDup.");
+})();
+})();
diff --git a/jstests/disk/too_many_fds.js b/jstests/disk/too_many_fds.js
index 68d70396900..d442ae5fab4 100644
--- a/jstests/disk/too_many_fds.js
+++ b/jstests/disk/too_many_fds.js
@@ -1,14 +1,20 @@
 // Create more than 1024 files on certain storage engines, then restart the server and see that it
 // can still listen on fd's smaller than FD_SETSIZE.
 
-function doTest() {
+(function doTest() {
     var baseName = "jstests_disk_too_many_fds";
 
     var m = MongoRunner.runMongod();
+    var db = m.getDB("db");
+    if (db.serverBuildInfo().bits != 64) {
+        print("Skipping.  Only run this test on 64bit builds");
+        return;
+    }
+
     // Make 1026 collections, each in a separate database.  On some storage engines, this may cause
     // 1026 files to be created.
     for (var i = 1; i < 1026; ++i) {
-        var db = m.getDB("db" + i);
+        db = m.getDB("db" + i);
         var coll = db.getCollection("coll" + i);
         assert.commandWorked(coll.insert({}));
     }
@@ -23,10 +29,4 @@ function doTest() {
     // Remove the data files generated by this test to avoid filling up the disk and leaving no
     // space available to start mongod processes for subsequent tests.
     resetDbpath(m.dbpath);
-}
-
-if (db.serverBuildInfo().bits == 64) {
-    doTest();
-} else {
-    print("Skipping.  Only run this test on 64bit builds");
-}
+})();
diff --git a/jstests/disk/wt_repair_corrupt_metadata.js b/jstests/disk/wt_repair_corrupt_metadata.js
index 2f8edfe2608..46dd2539114 100644
--- a/jstests/disk/wt_repair_corrupt_metadata.js
+++ b/jstests/disk/wt_repair_corrupt_metadata.js
@@ -24,15 +24,6 @@ const dbpath = MongoRunner.dataPath + baseName + "/";
  * be recreated with all of its data.
  */
 let runTest = function(mongodOptions) {
-    // Unfortunately using --nojournal triggers a WT_PANIC and aborts in debug builds, which the
-    // following test case can exercise.
-    // TODO: This return can be removed once WT-4310 is completed.
-    let isDebug = db.adminCommand('buildInfo').debug;
-    if (isDebug) {
-        jsTestLog("Skipping test case because this is a debug build");
-        return;
-    }
-
     resetDbpath(dbpath);
     jsTestLog("Running test with args: " + tojson(mongodOptions));
 
@@ -40,6 +31,15 @@ let runTest = function(mongodOptions) {
     const turtleFileWithoutCollection = dbpath + "WiredTiger.turtle.1";
 
     let mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+    // Unfortunately using --nojournal triggers a WT_PANIC and aborts in debug builds, which the
+    // following test case can exercise.
+    // TODO: This return can be removed once WT-4310 is completed.
+    let isDebug = mongod.getDB(baseName).adminCommand('buildInfo').debug;
+    if (isDebug) {
+        jsTestLog("Skipping test case because this is a debug build");
+        MongoRunner.stopMongod(mongod);
+        return;
+    }
 
     // Force a checkpoint and make a copy of the turtle file.
     assert.commandWorked(mongod.getDB(baseName).adminCommand({fsync: 1}));
diff --git a/jstests/disk/wt_repair_inconsistent_index.js b/jstests/disk/wt_repair_inconsistent_index.js
index 249303e081e..85124f610df 100644
--- a/jstests/disk/wt_repair_inconsistent_index.js
+++ b/jstests/disk/wt_repair_inconsistent_index.js
@@ -12,11 +12,6 @@ const baseName = "wt_repair_inconsistent_index";
 const collName = "test";
 const dbpath = MongoRunner.dataPath + baseName + "/";
 
-const forceCheckpoint = () => {
-    assert.commandWorked(db.fsyncLock());
-    assert.commandWorked(db.fsyncUnlock());
-};
-
 /**
  * Run the test by supplying additional paramters to MongoRunner.runMongod with 'mongodOptions'.
  */
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript
index 0178b45e9a8..05eb9f9c3dc 100644
--- a/src/mongo/db/catalog/SConscript
+++ b/src/mongo/db/catalog/SConscript
@@ -118,8 +118,10 @@ env.Library(
         '$BUILD_DIR/mongo/base',
     ],
     LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/db/catalog_raii',
         '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception',
         '$BUILD_DIR/mongo/db/query/query_knobs',
+        '$BUILD_DIR/mongo/db/storage/storage_repair_observer',
     ]
 )
 
diff --git a/src/mongo/db/catalog/index_builds_manager.cpp b/src/mongo/db/catalog/index_builds_manager.cpp
index 1548bf51850..b4860cbfffe 100644
--- a/src/mongo/db/catalog/index_builds_manager.cpp
+++ b/src/mongo/db/catalog/index_builds_manager.cpp
@@ -37,8 +37,11 @@
 #include "mongo/db/catalog/collection_catalog.h"
 #include "mongo/db/catalog/index_catalog.h"
 #include "mongo/db/catalog/uncommitted_collections.h"
+#include "mongo/db/catalog_raii.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
+#include "mongo/db/namespace_string.h"
 #include "mongo/db/operation_context.h"
+#include "mongo/db/storage/storage_repair_observer.h"
 #include "mongo/db/storage/write_unit_of_work.h"
 #include "mongo/logv2/log.h"
 #include "mongo/util/assert_util.h"
@@ -210,11 +213,23 @@ StatusWith<std::pair<long long, long long>> IndexBuildsManager::startBuildingInd
     }
 
     progressMeter.finished();
+    std::set<RecordId> dups;
 
-    Status status = builder->dumpInsertsFromBulk(opCtx);
+    Status status =
+        builder->dumpInsertsFromBulk(opCtx, (repair == RepairData::kYes) ? &dups : nullptr);
     if (!status.isOK()) {
         return status;
     }
+
+    // Delete duplicate documents and insert them into local lost and found.
+    // TODO SERVER-49507: Reduce memory consumption when there are a large number of duplicate
+    // records.
+    auto dupRecordsDeleted = _moveDocsToLostAndFound(opCtx, ns, &dups);
+    if (!dupRecordsDeleted.isOK()) {
+        return dupRecordsDeleted.getStatus();
+    }
+    numRecords -= dupRecordsDeleted.getValue();
+
     return std::make_pair(numRecords, dataSize);
 }
 
@@ -351,4 +366,91 @@ StatusWith<MultiIndexBlock*> IndexBuildsManager::_getBuilder(const UUID& buildUU
     return builderIt->second.get();
 }
 
+StatusWith<long long> IndexBuildsManager::_moveDocsToLostAndFound(OperationContext* opCtx,
+                                                                  NamespaceString nss,
+                                                                  std::set<RecordId>* dupRecords) {
+    invariant(opCtx->lockState()->isCollectionLockedForMode(nss, MODE_IX));
+
+    long long recordsDeleted = 0;
+    if (dupRecords->empty()) {
+        return recordsDeleted;
+    }
+
+    auto originalCollection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss);
+    auto collUUID = originalCollection->uuid();
+
+    const NamespaceString lostAndFoundNss =
+        NamespaceString(NamespaceString::kLocalDb, "system.lost_and_found." + collUUID.toString());
+    Collection* localCollection =
+        CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, lostAndFoundNss);
+
+    // Create the collection if it doesn't exist.
+    if (!localCollection) {
+        Status status =
+            writeConflictRetry(opCtx, "createLostAndFoundCollection", lostAndFoundNss.ns(), [&]() {
+                WriteUnitOfWork wuow(opCtx);
+                AutoGetOrCreateDb autoDb(opCtx, NamespaceString::kLocalDb, MODE_X);
+                Database* db = autoDb.getDb();
+
+                // Ensure the database exists.
+                invariant(db);
+
+                // Since we are potentially deleting documents with duplicate _id values, we need to
+                // be able to insert into the lost and found collection without generating any
+                // duplicate key errors on the _id value.
+                CollectionOptions collOptions;
+                collOptions.setNoIdIndex();
+                localCollection = db->createCollection(opCtx, lostAndFoundNss, collOptions);
+
+                // Ensure the collection exists.
+                invariant(localCollection);
+                wuow.commit();
+                return Status::OK();
+            });
+        if (!status.isOK()) {
+            return status;
+        }
+    }
+
+    for (auto&& recordId : *dupRecords) {
+
+        Status status =
+            writeConflictRetry(opCtx, "writeDupDocToLostAndFoundCollection", nss.ns(), [&]() {
+                WriteUnitOfWork wuow(opCtx);
+
+                Snapshotted<BSONObj> doc;
+                // If the record doesn’t exist, continue with other documents.
+                if (!originalCollection->findDoc(opCtx, recordId, &doc)) {
+                    return Status::OK();
+                }
+
+                // Write document to lost_and_found collection and delete from original collection.
+                Status status =
+                    localCollection->insertDocument(opCtx, InsertStatement(doc.value()), nullptr);
+                if (!status.isOK()) {
+                    return status;
+                }
+
+                originalCollection->deleteDocument(opCtx, kUninitializedStmtId, recordId, nullptr);
+                wuow.commit();
+
+                recordsDeleted++;
+                return Status::OK();
+            });
+        if (!status.isOK()) {
+            return status;
+        }
+    }
+    StorageRepairObserver::get(opCtx->getServiceContext())
+        ->invalidatingModification(str::stream()
+                                   << "Moved " << recordsDeleted
+                                   << " docs to lost and found: " << localCollection->ns());
+
+    LOGV2(3956200,
+          "Moved documents to lost and found.",
+          "numDocuments"_attr = recordsDeleted,
+          "lostAndFoundNss"_attr = lostAndFoundNss);
+    return recordsDeleted;
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/catalog/index_builds_manager.h b/src/mongo/db/catalog/index_builds_manager.h
index bfcd8d77c12..e7fc7644d95 100644
--- a/src/mongo/db/catalog/index_builds_manager.h
+++ b/src/mongo/db/catalog/index_builds_manager.h
@@ -200,6 +200,14 @@ private:
     // Map of index builders by build UUID. Allows access to the builders so that actions can be
     // taken on and information passed to and from index builds.
     std::map<UUID, std::unique_ptr<MultiIndexBlock>> _builders;
+
+    /**
+     * Deletes records containing duplicate keys and inserts them into a local lost and found
+     * collection titled "local.system.lost_and_found.<original collection UUID>".
+     */
+    StatusWith<long long> _moveDocsToLostAndFound(OperationContext* opCtx,
+                                                  NamespaceString ns,
+                                                  std::set<RecordId>* dupRecords);
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp
index b5cfff81695..3835961496c 100644
--- a/src/mongo/db/index_builds_coordinator.cpp
+++ b/src/mongo/db/index_builds_coordinator.cpp
@@ -2512,6 +2512,14 @@ StatusWith<std::pair<long long, long long>> IndexBuildsCoordinator::_runIndexReb
             uassertStatusOK(_indexBuildsManager.startBuildingIndexForRecovery(
                 opCtx, collection->ns(), buildUUID, repair));
 
+        // Since we are holding an exclusive collection lock to stop new writes, do not yield locks
+        // while draining.
+        uassertStatusOK(_indexBuildsManager.drainBackgroundWrites(
+            opCtx,
+            replState->buildUUID,
+            RecoveryUnit::ReadSource::kUnset,
+            IndexBuildInterceptor::DrainYieldPolicy::kNoYield));
+
         uassertStatusOK(
             _indexBuildsManager.checkIndexConstraintViolations(opCtx, replState->buildUUID));