SERVER-61427 Skip duplicate index key handler when exact key already exists in index

(cherry picked from commit 9f15b2b1eeb356f4e3d0c6e06302941ec685cc0a)
(cherry picked from commit cbfe08edee4a6aa41694b73be690b4e56de0bb3a)
(cherry picked from commit e729f76f2c5de716e0724651e14a51b7e424258a)

6 files changed, 69 insertions, 21 deletions

diff --git a/jstests/noPassthrough/unique_index_insert_during_collection_scan.js b/jstests/noPassthrough/unique_index_insert_during_collection_scan.js
new file mode 100644
index 00000000000..e3c0ce98af4
--- /dev/null
+++ b/jstests/noPassthrough/unique_index_insert_during_collection_scan.js
@@ -0,0 +1,36 @@
+/**
+ * Tests inserting into collection while a unique index build is in the collection scan phase.
+ * Ensures that even though the insert is seen by both the collection scan and the side writes
+ * table, the index build does not need to resolve any duplicate keys.
+ */
+(function() {
+"use strict";
+
+load('jstests/libs/fail_point_util.js');
+load('jstests/libs/parallel_shell_helpers.js');
+
+const conn = MongoRunner.runMongod();
+const coll = conn.getDB('test')[jsTestName()];
+
+assert.commandWorked(coll.insert({_id: 0, a: 0}));
+
+const fp = configureFailPoint(conn, 'hangAfterSettingUpIndexBuild');
+const awaitCreateIndex =
+    startParallelShell(funWithArgs(function(collName) {
+                           assert.commandWorked(db[collName].createIndex({a: 1}, {unique: true}));
+                       }, coll.getName()), conn.port);
+fp.wait();
+assert.commandWorked(coll.insert({_id: 1, a: 1}));
+fp.off();
+
+awaitCreateIndex();
+
+// Ensure that the second document was seen by both the collection scan and the side writes table.
+checkLog.contains(conn, 'index build: inserted 2 keys from external sorter into index');
+checkLog.contains(conn, 'index build: drain applied 1 side writes');
+
+// Ensure that there were no duplicates to resolve.
+assert(!checkLog.checkContainsOnce(conn, 'duplicate key conflicts for unique index'));
+
+MongoRunner.stopMongod(conn);
+})();
diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp
index b9024a5fd94..41ff27a6ddb 100644
--- a/src/mongo/db/index/index_access_method.cpp
+++ b/src/mongo/db/index/index_access_method.cpp
@@ -264,8 +264,13 @@ Status AbstractIndexAccessMethod::insertKeys(OperationContext* opCtx,
                 ret = _newInterface->insert(opCtx, key, loc, true /* dupsAllowed */);
                 status = ret.getStatus();
 
-                if (status.isOK() && onDuplicateKey)
+                if (status.isOK() && ret.getValue() != SpecialFormatInserted::NothingInserted &&
+                    onDuplicateKey) {
+                    // Only run the duplicate key handler if we inserted the key ourselves. Someone
+                    // else could have already inserted this exact key, but in that case we don't
+                    // count it as a duplicate.
                     status = onDuplicateKey(key);
+                }
             }
 
             if (status.isOK() && ret.getValue() == SpecialFormatInserted::LongTypeBitsInserted)
diff --git a/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp b/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp
index 32f75c0ee6f..5946243cee2 100644
--- a/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp
+++ b/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp
@@ -329,8 +329,7 @@ StatusWith<SpecialFormatInserted> SortedDataInterface::insert(OperationContext*
                         key, _collectionNamespace, _indexName, _keyPattern, _collation);
                 }
             } else {
-                return StatusWith<SpecialFormatInserted>(
-                    SpecialFormatInserted::NoSpecialFormatInserted);
+                return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::NothingInserted);
             }
         }
     } else {
@@ -338,7 +337,7 @@ StatusWith<SpecialFormatInserted> SortedDataInterface::insert(OperationContext*
     }
 
     if (workingCopy->find(insertKeyString) != workingCopy->end())
-        return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::NoSpecialFormatInserted);
+        return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::NothingInserted);
 
     // The value we insert is the RecordId followed by the typebits.
     std::string internalTbString = std::string(workingCopyInternalKs->getTypeBits().getBuffer(),
diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp
index 3b5f6f2cd02..536aa443315 100644
--- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp
+++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp
@@ -189,8 +189,10 @@ public:
         if (_data->insert(entry).second) {
             _currentKeySize += key.objsize();
             opCtx->recoveryUnit()->registerChange(new IndexChange(_data, entry, true));
+            return StatusWith<SpecialFormatInserted>(
+                SpecialFormatInserted::NoSpecialFormatInserted);
         }
-        return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::NoSpecialFormatInserted);
+        return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::NothingInserted);
     }
 
     virtual void unindex(OperationContext* opCtx,
diff --git a/src/mongo/db/storage/sorted_data_interface.h b/src/mongo/db/storage/sorted_data_interface.h
index ee745bf7cfe..3f5d1b91a63 100644
--- a/src/mongo/db/storage/sorted_data_interface.h
+++ b/src/mongo/db/storage/sorted_data_interface.h
@@ -50,7 +50,11 @@ struct ValidateResults;
  * is a way to inform the callers to do something when special format exists on disk.
  * TODO SERVER-36385: Remove this enum in 4.4.
  */
-enum SpecialFormatInserted { NoSpecialFormatInserted = 0, LongTypeBitsInserted = 1 };
+enum SpecialFormatInserted {
+    NothingInserted = 0,
+    NoSpecialFormatInserted = 1,
+    LongTypeBitsInserted = 2,
+};
 
 /**
  * This is the uniform interface for storing indexes and supporting point queries as well as range
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index 9c1da2c5a3d..472269c05be 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -1441,16 +1441,13 @@ StatusWith<SpecialFormatInserted> WiredTigerIndexUnique::_insertTimestampUnsafe(
     c->set_value(c, valueItem.Get());
     int ret = WT_OP_CHECK(c->insert(c));
 
-    if (ret != WT_DUPLICATE_KEY) {
-        if (ret == 0) {
-            if (data.getTypeBits().isLongEncoding())
-                return StatusWith<SpecialFormatInserted>(
-                    SpecialFormatInserted::LongTypeBitsInserted);
-            else
-                return StatusWith<SpecialFormatInserted>(
-                    SpecialFormatInserted::NoSpecialFormatInserted);
-        }
-
+    if (!ret) {
+        if (data.getTypeBits().isLongEncoding())
+            return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::LongTypeBitsInserted);
+        else
+            return StatusWith<SpecialFormatInserted>(
+                SpecialFormatInserted::NoSpecialFormatInserted);
+    } else if (ret != WT_DUPLICATE_KEY) {
         return wtRCToStatus(ret);
     }
 
@@ -1472,7 +1469,7 @@ StatusWith<SpecialFormatInserted> WiredTigerIndexUnique::_insertTimestampUnsafe(
         RecordId idInIndex = KeyString::decodeRecordId(&br);
         if (id == idInIndex)
             return StatusWith<SpecialFormatInserted>(
-                SpecialFormatInserted::NoSpecialFormatInserted);  // already in index
+                SpecialFormatInserted::NothingInserted);  // already in index
 
         if (!insertedId && id < idInIndex) {
             value.appendRecordId(id);
@@ -1563,8 +1560,10 @@ StatusWith<SpecialFormatInserted> WiredTigerIndexUnique::_insertTimestampSafe(
     ret = WT_OP_CHECK(c->insert(c));
 
     // It is possible that this key is already present during a concurrent background index build.
-    if (ret != WT_DUPLICATE_KEY)
-        invariantWTOK(ret);
+    if (ret == WT_DUPLICATE_KEY) {
+        return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::NothingInserted);
+    }
+    invariantWTOK(ret);
 
     if (tableKey.getTypeBits().isLongEncoding())
         return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::LongTypeBitsInserted);
@@ -1768,11 +1767,14 @@ StatusWith<SpecialFormatInserted> WiredTigerIndexStandard::_insert(OperationCont
     c->set_value(c, valueItem.Get());
     int ret = WT_OP_CHECK(c->insert(c));
 
-    // If the record was already in the index, we just return OK.
+    // If the record was already in the index, return NothingInserted.
     // This can happen, for example, when building a background index while documents are being
     // written and reindexed.
-    if (ret != 0 && ret != WT_DUPLICATE_KEY)
+    if (ret == WT_DUPLICATE_KEY) {
+        return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::NothingInserted);
+    } else if (ret) {
         return wtRCToStatus(ret);
+    }
 
     if (key.getTypeBits().isLongEncoding())
         return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::LongTypeBitsInserted);