SERVER-44571 Documents involved in SERVER-44050 corruption scenario cannot be updated or deleted after upgrade

(cherry picked from commit 35c6778143fc55eb9617ab4a54e616ba1e537ad5)

25 files changed, 272 insertions, 28 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index da1de4230d9..85166fdda8f 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -2058,7 +2058,7 @@ functions:
           --edition ${multiversion_edition|base}             \
           --platform ${multiversion_platform|linux}          \
           --architecture ${multiversion_architecture|x86_64} \
-          --useLatest 3.2 3.4 3.6 4.0 4.0.1 4.0.5
+          --useLatest 3.2 3.4 3.6 4.0 4.0.1 4.0.5 4.2.1
 
   "do snmp setup":
     command: shell.exec
diff --git a/jstests/multiVersion/hashed_index_bad_keys_cleanup.js b/jstests/multiVersion/hashed_index_bad_keys_cleanup.js
new file mode 100644
index 00000000000..39675fdb730
--- /dev/null
+++ b/jstests/multiVersion/hashed_index_bad_keys_cleanup.js
@@ -0,0 +1,122 @@
+/**
+ * Prior to SERVER-44050, for hashed indexes, if there is an array along index field path, we did
+ * not fail insertion. We incorrectly stored empty index key value for those cases. This lead to
+ * corruption of index keys.
+ *
+ * In this test we verify that we are able to successfully update and delete documents that were
+ * involved in creating corrupt indexes.
+ *
+ * When we branch for 4.4, this test should be deleted in the master branch. In the v4.4 branch it
+ * should test a 4.2 => 4.4 upgrade scenario.
+ */
+(function() {
+"use strict";
+
+load("jstests/multiVersion/libs/multi_rs.js");         // For upgradeSet.
+load("jstests/multiVersion/libs/verify_versions.js");  // For binVersion.
+
+const preBackportVersion = "4.2.1";
+const preBackportNodeOptions = {
+    binVersion: preBackportVersion
+};
+const nodeOptionsOfLatestVersion = {
+    binVersion: "latest"
+};
+
+// Set up a new replSet consisting of 2 nodes, initially running on bad binaries.
+const rst = new ReplSetTest({nodes: 2, nodeOptions: preBackportNodeOptions});
+rst.startSet();
+rst.initiate();
+
+let testDB = rst.getPrimary().getDB(jsTestName());
+let coll = testDB.coll;
+coll.drop();
+
+// Verify that the replset is on binary version specified in 'preBackportVersion'.
+assert.binVersion(testDB.getMongo(), preBackportVersion);
+
+// Insert bad documents using older version.
+assert.commandWorked(coll.createIndex({"p.q.r": "hashed"}));
+assert.commandWorked(coll.insert({_id: 1, p: []}));
+assert.commandWorked(coll.insert({_id: 2, p: {q: [1]}}));
+assert.commandWorked(coll.insert({_id: 3, p: [{q: 1}]}));
+assert.commandWorked(coll.insert({_id: 4, a: 1, p: [{q: 1}]}));
+assert.commandWorked(coll.insert({_id: 5, a: 1, p: [{q: 1}]}));
+
+// Helper function which runs validate() on primary and secondary nodes, then verifies that the
+// command returned the expected result.
+function assertValidateCmdReturned(expectedResult) {
+    const resFromPrimary = assert.commandWorked(coll.validate({full: true}));
+    assert.eq(resFromPrimary.valid, expectedResult, resFromPrimary);
+
+    rst.awaitReplication();
+    const testDBOnSecondary = rst.getSecondary().getDB(jsTestName());
+    const resFromSecondary = assert.commandWorked(testDBOnSecondary.coll.validate({full: true}));
+    assert.eq(resFromSecondary.valid, expectedResult, resFromSecondary);
+}
+
+// Confirm that validate() does not perceive a problem with the malformed documents.
+assertValidateCmdReturned(true);
+
+// Upgrade the set to the new binary version.
+rst.upgradeSet(nodeOptionsOfLatestVersion);
+testDB = rst.getPrimary().getDB(jsTestName());
+coll = testDB.coll;
+
+// Verify that the five documents inserted earlier have their index keys.
+let res = coll.find().hint({"p.q.r": "hashed"}).returnKey().itcount();
+assert.eq(res, 5);
+
+// Verify that after upgrade, inserting bad documents is not allowed.
+const arrayAlongPathFailCode = 16766;
+assert.commandFailedWithCode(coll.insert({p: []}), arrayAlongPathFailCode);
+assert.commandFailedWithCode(coll.insert({p: [{q: 1}]}), arrayAlongPathFailCode);
+assert.commandFailedWithCode(coll.insert({p: {q: {r: [3]}}}), arrayAlongPathFailCode);
+
+// After upgrade, validate() should now fail since there are existing bad documents.
+assertValidateCmdReturned(false);
+
+// Deleting bad documents succeeds.
+assert.commandWorked(coll.deleteOne({_id: 1}));
+assert.commandWorked(coll.deleteMany({a: 1}));
+
+// Updating documents to contain array along field path should fail.
+assert.commandFailedWithCode(coll.update({_id: 2}, {p: {q: [{r: 1}]}}), arrayAlongPathFailCode);
+assert.commandFailedWithCode(
+    testDB.runCommand({findAndModify: coll.getName(), query: {_id: 2}, update: {p: {q: [{r: 1}]}}}),
+    arrayAlongPathFailCode);
+
+assert.commandFailedWithCode(coll.update({_id: 2}, {p: {q: {r: [3]}}}), arrayAlongPathFailCode);
+assert.commandFailedWithCode(testDB.runCommand({
+    update: coll.getName(),
+    updates: [
+        {q: {_id: 3}, u: {$set: {p: {q: [{r: 1}]}}}},
+        {q: {_id: 2}, u: {$set: {p: {q: [{r: 1}]}}}}
+    ],
+    ordered: false
+}),
+                             arrayAlongPathFailCode);
+
+// Verify that updating to a valid index field works.
+assert.commandWorked(coll.update({_id: 2}, {p: {q: {r: 4}}}));
+
+// Verify that the index key is updated correctly by quering with hashed index.
+res = coll.find({"p.q.r": 4}).hint({"p.q.r": "hashed"}).toArray();
+assert.eq(res, [{_id: 2, p: {q: {r: 4}}}]);
+
+// Validate should still fail since a bad document {_id: 3} exists.
+assertValidateCmdReturned(false);
+
+// Delete the last remaining bad document.
+assert.commandWorked(coll.deleteOne({_id: 3}));
+
+// Now that all the bad documents are deleted or updated, verify that validate succeeds.
+assertValidateCmdReturned(true);
+
+// Verify that there is only one index key left (for {_id: 2}).
+res = coll.find().hint({"p.q.r": "hashed"}).returnKey().itcount();
+assert.eq(res, 1);
+
+rst.awaitReplication();
+rst.stopSet();
+}());
\ No newline at end of file
diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp
index 6dc6672782f..13c194b6c98 100644
--- a/src/mongo/db/catalog/index_catalog_impl.cpp
+++ b/src/mongo/db/catalog/index_catalog_impl.cpp
@@ -1349,8 +1349,12 @@ Status IndexCatalogImpl::_indexFilteredRecords(OperationContext* opCtx,
         BSONObjSet multikeyMetadataKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
         MultikeyPaths multikeyPaths;
 
-        index->accessMethod()->getKeys(
-            *bsonRecord.docPtr, options.getKeysMode, &keys, &multikeyMetadataKeys, &multikeyPaths);
+        index->accessMethod()->getKeys(*bsonRecord.docPtr,
+                                       options.getKeysMode,
+                                       IndexAccessMethod::GetKeysContext::kReadOrAddKeys,
+                                       &keys,
+                                       &multikeyMetadataKeys,
+                                       &multikeyPaths);
 
         Status status = _indexKeys(opCtx,
                                    index,
@@ -1504,8 +1508,12 @@ void IndexCatalogImpl::_unindexRecord(OperationContext* opCtx,
     // deleted.
     BSONObjSet keys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
 
-    entry->accessMethod()->getKeys(
-        obj, IndexAccessMethod::GetKeysMode::kRelaxConstraintsUnfiltered, &keys, nullptr, nullptr);
+    entry->accessMethod()->getKeys(obj,
+                                   IndexAccessMethod::GetKeysMode::kRelaxConstraintsUnfiltered,
+                                   IndexAccessMethod::GetKeysContext::kRemovingKeys,
+                                   &keys,
+                                   nullptr,
+                                   nullptr);
 
     _unindexKeys(opCtx, entry, {keys.begin(), keys.end()}, obj, loc, logIfError, keysDeletedOut);
 }
diff --git a/src/mongo/db/catalog/private/record_store_validate_adaptor.cpp b/src/mongo/db/catalog/private/record_store_validate_adaptor.cpp
index 4e2c4e73a48..db316c4a098 100644
--- a/src/mongo/db/catalog/private/record_store_validate_adaptor.cpp
+++ b/src/mongo/db/catalog/private/record_store_validate_adaptor.cpp
@@ -101,6 +101,7 @@ Status RecordStoreValidateAdaptor::validate(const RecordId& recordId,
         MultikeyPaths multikeyPaths;
         iam->getKeys(recordBson,
                      IndexAccessMethod::GetKeysMode::kEnforceConstraints,
+                     IndexAccessMethod::GetKeysContext::kReadOrAddKeys,
                      &documentKeySet,
                      &multikeyMetadataKeys,
                      &multikeyPaths);
diff --git a/src/mongo/db/exec/working_set_common.cpp b/src/mongo/db/exec/working_set_common.cpp
index c3d65fa448b..354feb81479 100644
--- a/src/mongo/db/exec/working_set_common.cpp
+++ b/src/mongo/db/exec/working_set_common.cpp
@@ -87,6 +87,7 @@ bool WorkingSetCommon::fetch(OperationContext* opCtx,
             MultikeyPaths* multikeyPaths = nullptr;
             member->keyData[i].index->getKeys(member->obj.value(),
                                               IndexAccessMethod::GetKeysMode::kEnforceConstraints,
+                                              IndexAccessMethod::GetKeysContext::kReadOrAddKeys,
                                               &keys,
                                               multikeyMetadataKeys,
                                               multikeyPaths);
diff --git a/src/mongo/db/index/2d_access_method.cpp b/src/mongo/db/index/2d_access_method.cpp
index 99f52b5e5ed..6e2a8212fd7 100644
--- a/src/mongo/db/index/2d_access_method.cpp
+++ b/src/mongo/db/index/2d_access_method.cpp
@@ -50,6 +50,7 @@ TwoDAccessMethod::TwoDAccessMethod(IndexCatalogEntry* btreeState, SortedDataInte
 
 /** Finds the key objects to put in an index */
 void TwoDAccessMethod::doGetKeys(const BSONObj& obj,
+                                 GetKeysContext context,
                                  BSONObjSet* keys,
                                  BSONObjSet* multikeyMetadataKeys,
                                  MultikeyPaths* multikeyPaths) const {
diff --git a/src/mongo/db/index/2d_access_method.h b/src/mongo/db/index/2d_access_method.h
index 0d350f99869..38e5ebdda3f 100644
--- a/src/mongo/db/index/2d_access_method.h
+++ b/src/mongo/db/index/2d_access_method.h
@@ -59,6 +59,7 @@ private:
      * indexes don't support tracking path-level multikey information.
      */
     void doGetKeys(const BSONObj& obj,
+                   GetKeysContext context,
                    BSONObjSet* keys,
                    BSONObjSet* multikeyMetadataKeys,
                    MultikeyPaths* multikeyPaths) const final;
diff --git a/src/mongo/db/index/btree_access_method.cpp b/src/mongo/db/index/btree_access_method.cpp
index c96fcfc1dc8..d568fb3200e 100644
--- a/src/mongo/db/index/btree_access_method.cpp
+++ b/src/mongo/db/index/btree_access_method.cpp
@@ -60,6 +60,7 @@ BtreeAccessMethod::BtreeAccessMethod(IndexCatalogEntry* btreeState, SortedDataIn
 }
 
 void BtreeAccessMethod::doGetKeys(const BSONObj& obj,
+                                  GetKeysContext context,
                                   BSONObjSet* keys,
                                   BSONObjSet* multikeyMetadataKeys,
                                   MultikeyPaths* multikeyPaths) const {
diff --git a/src/mongo/db/index/btree_access_method.h b/src/mongo/db/index/btree_access_method.h
index 126c1fa86eb..c81e454d04c 100644
--- a/src/mongo/db/index/btree_access_method.h
+++ b/src/mongo/db/index/btree_access_method.h
@@ -49,6 +49,7 @@ public:
 
 private:
     void doGetKeys(const BSONObj& obj,
+                   GetKeysContext context,
                    BSONObjSet* keys,
                    BSONObjSet* multikeyMetadataKeys,
                    MultikeyPaths* multikeyPaths) const final;
diff --git a/src/mongo/db/index/expression_keys_private.cpp b/src/mongo/db/index/expression_keys_private.cpp
index cd170c24819..18623088ede 100644
--- a/src/mongo/db/index/expression_keys_private.cpp
+++ b/src/mongo/db/index/expression_keys_private.cpp
@@ -49,6 +49,7 @@
 #include "mongo/util/assert_util.h"
 #include "mongo/util/log.h"
 #include "mongo/util/str.h"
+
 #include "third_party/s2/s2cell.h"
 #include "third_party/s2/s2regioncoverer.h"
 
@@ -344,20 +345,35 @@ void ExpressionKeysPrivate::getHashKeys(const BSONObj& obj,
                                         int hashVersion,
                                         bool isSparse,
                                         const CollatorInterface* collator,
-                                        BSONObjSet* keys) {
+                                        BSONObjSet* keys,
+                                        bool ignoreArraysAlongPath) {
+    static const BSONObj nullObj = BSON("" << BSONNULL);
     const char* cstr = hashedField.c_str();
     BSONElement fieldVal = dps::extractElementAtPathOrArrayAlongPath(obj, cstr);
 
     // If we hit an array while traversing the path, 'cstr' will point to the path component
-    // immediately following the array, or the null termination byte if the final field in the path
-    // was an array.
-    uassert(
-        16766,
-        str::stream()
-            << "Error: hashed indexes do not currently support array values. Found array at path: "
-            << hashedField.substr(
-                   0, hashedField.size() - StringData(cstr).size() - !StringData(cstr).empty()),
-        fieldVal.type() != BSONType::Array);
+    // immediately following the array, or the null termination byte if the terminal path
+    // component was an array. In the latter case, 'remainingPath' will be empty.
+    auto remainingPath = StringData(cstr);
+
+    // If 'ignoreArraysAlongPath' is set, we want to use the behaviour prior to SERVER-44050,
+    // which is to allow arrays along the field path (except the terminal path). This is done so
+    // that the document keys inserted prior to SERVER-44050 can be deleted or updated after the
+    // upgrade, allowing users to recover from the possible index corruption. The old behaviour
+    // before SERVER-44050 was to store 'null' index key if we encountered an array along the
+    // index field path. We will use the same logic in the context of removing index keys.
+    if (ignoreArraysAlongPath && fieldVal.type() == BSONType::Array && !remainingPath.empty()) {
+        fieldVal = nullObj.firstElement();
+    }
+
+    // Otherwise, throw if an array was encountered at any point along the path.
+    uassert(16766,
+            str::stream() << "Error: hashed indexes do not currently support array values. "
+                             "Found array at path: "
+                          << hashedField.substr(0,
+                                                hashedField.size() - remainingPath.size() -
+                                                    !remainingPath.empty()),
+            fieldVal.type() != BSONType::Array);
 
     // Convert strings to comparison keys.
     BSONObj fieldValObj;
@@ -369,7 +385,6 @@ void ExpressionKeysPrivate::getHashKeys(const BSONObj& obj,
         BSONObj key = BSON("" << makeSingleHashKey(fieldVal, seed, hashVersion));
         keys->insert(key);
     } else if (!isSparse) {
-        BSONObj nullObj = BSON("" << BSONNULL);
         keys->insert(BSON("" << makeSingleHashKey(nullObj.firstElement(), seed, hashVersion)));
     }
 }
diff --git a/src/mongo/db/index/expression_keys_private.h b/src/mongo/db/index/expression_keys_private.h
index b5fbed56e70..be8767e4296 100644
--- a/src/mongo/db/index/expression_keys_private.h
+++ b/src/mongo/db/index/expression_keys_private.h
@@ -80,7 +80,8 @@ public:
                             int hashVersion,
                             bool isSparse,
                             const CollatorInterface* collator,
-                            BSONObjSet* keys);
+                            BSONObjSet* keys,
+                            bool ignoreArraysAlongPath);
 
     /**
      * Hashing function used by both getHashKeys and the cursors we create.
diff --git a/src/mongo/db/index/fts_access_method.cpp b/src/mongo/db/index/fts_access_method.cpp
index 2912569dfb4..13a686aa8cd 100644
--- a/src/mongo/db/index/fts_access_method.cpp
+++ b/src/mongo/db/index/fts_access_method.cpp
@@ -38,6 +38,7 @@ FTSAccessMethod::FTSAccessMethod(IndexCatalogEntry* btreeState, SortedDataInterf
     : AbstractIndexAccessMethod(btreeState, btree), _ftsSpec(btreeState->descriptor()->infoObj()) {}
 
 void FTSAccessMethod::doGetKeys(const BSONObj& obj,
+                                GetKeysContext context,
                                 BSONObjSet* keys,
                                 BSONObjSet* multikeyMetadataKeys,
                                 MultikeyPaths* multikeyPaths) const {
diff --git a/src/mongo/db/index/fts_access_method.h b/src/mongo/db/index/fts_access_method.h
index dbd0a6eb175..50aa87d548e 100644
--- a/src/mongo/db/index/fts_access_method.h
+++ b/src/mongo/db/index/fts_access_method.h
@@ -53,6 +53,7 @@ private:
      * indexes don't support tracking path-level multikey information.
      */
     void doGetKeys(const BSONObj& obj,
+                   GetKeysContext context,
                    BSONObjSet* keys,
                    BSONObjSet* multikeyMetadataKeys,
                    MultikeyPaths* multikeyPaths) const final;
diff --git a/src/mongo/db/index/hash_access_method.cpp b/src/mongo/db/index/hash_access_method.cpp
index c78a3f1cf9f..d7d91f48239 100644
--- a/src/mongo/db/index/hash_access_method.cpp
+++ b/src/mongo/db/index/hash_access_method.cpp
@@ -55,11 +55,18 @@ HashAccessMethod::HashAccessMethod(IndexCatalogEntry* btreeState, SortedDataInte
 }
 
 void HashAccessMethod::doGetKeys(const BSONObj& obj,
+                                 GetKeysContext context,
                                  BSONObjSet* keys,
                                  BSONObjSet* multikeyMetadataKeys,
                                  MultikeyPaths* multikeyPaths) const {
-    ExpressionKeysPrivate::getHashKeys(
-        obj, _hashedField, _seed, _hashVersion, _descriptor->isSparse(), _collator, keys);
+    ExpressionKeysPrivate::getHashKeys(obj,
+                                       _hashedField,
+                                       _seed,
+                                       _hashVersion,
+                                       _descriptor->isSparse(),
+                                       _collator,
+                                       keys,
+                                       (context == GetKeysContext::kRemovingKeys));
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/index/hash_access_method.h b/src/mongo/db/index/hash_access_method.h
index a11403ff784..4c41368630c 100644
--- a/src/mongo/db/index/hash_access_method.h
+++ b/src/mongo/db/index/hash_access_method.h
@@ -56,6 +56,7 @@ private:
      * indexes don't support tracking path-level multikey information.
      */
     void doGetKeys(const BSONObj& obj,
+                   GetKeysContext context,
                    BSONObjSet* keys,
                    BSONObjSet* multikeyMetadataKeys,
                    MultikeyPaths* multikeyPaths) const final;
diff --git a/src/mongo/db/index/hash_key_generator_test.cpp b/src/mongo/db/index/hash_key_generator_test.cpp
index e66e2bef5f6..2651f44d9cc 100644
--- a/src/mongo/db/index/hash_key_generator_test.cpp
+++ b/src/mongo/db/index/hash_key_generator_test.cpp
@@ -89,7 +89,7 @@ TEST(HashKeyGeneratorTest, CollationAppliedBeforeHashing) {
     BSONObjSet actualKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
     CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString);
     ExpressionKeysPrivate::getHashKeys(
-        obj, "a", kHashSeed, kHashVersion, false, &collator, &actualKeys);
+        obj, "a", kHashSeed, kHashVersion, false, &collator, &actualKeys, false);
 
     BSONObj backwardsObj = fromjson("{a: 'gnirts'}");
     BSONObjSet expectedKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
@@ -103,7 +103,7 @@ TEST(HashKeyGeneratorTest, CollationDoesNotAffectNonStringFields) {
     BSONObjSet actualKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
     CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString);
     ExpressionKeysPrivate::getHashKeys(
-        obj, "a", kHashSeed, kHashVersion, false, &collator, &actualKeys);
+        obj, "a", kHashSeed, kHashVersion, false, &collator, &actualKeys, false);
 
     BSONObjSet expectedKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
     expectedKeys.insert(makeHashKey(obj["a"]));
@@ -117,7 +117,7 @@ TEST(HashKeyGeneratorTest, CollatorAppliedBeforeHashingNestedObject) {
     BSONObjSet actualKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
     CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString);
     ExpressionKeysPrivate::getHashKeys(
-        obj, "a", kHashSeed, kHashVersion, false, &collator, &actualKeys);
+        obj, "a", kHashSeed, kHashVersion, false, &collator, &actualKeys, false);
 
     BSONObjSet expectedKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
     expectedKeys.insert(makeHashKey(backwardsObj["a"]));
@@ -129,7 +129,7 @@ TEST(HashKeyGeneratorTest, NoCollation) {
     BSONObj obj = fromjson("{a: 'string'}");
     BSONObjSet actualKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
     ExpressionKeysPrivate::getHashKeys(
-        obj, "a", kHashSeed, kHashVersion, false, nullptr, &actualKeys);
+        obj, "a", kHashSeed, kHashVersion, false, nullptr, &actualKeys, false);
 
     BSONObjSet expectedKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
     expectedKeys.insert(makeHashKey(obj["a"]));
@@ -137,4 +137,48 @@ TEST(HashKeyGeneratorTest, NoCollation) {
     ASSERT(assertKeysetsEqual(expectedKeys, actualKeys));
 }
 
+TEST(HashKeyGeneratorTest, ArrayAlongIndexFieldPathFails) {
+    BSONObj obj = fromjson("{a: []}");
+    BSONObjSet actualKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
+    ASSERT_THROWS_CODE(
+        ExpressionKeysPrivate::getHashKeys(
+            obj, "a.b.c", kHashSeed, kHashVersion, false, nullptr, &actualKeys, false),
+        DBException,
+        16766);
+}
+
+TEST(HashKeyGeneratorTest, ArrayAlongIndexFieldPathDoesNotFailWhenIgnoreFlagIsSet) {
+    BSONObj obj = fromjson("{a: []}");
+    BSONObjSet actualKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
+    ExpressionKeysPrivate::getHashKeys(obj,
+                                       "a.b.c",
+                                       kHashSeed,
+                                       kHashVersion,
+                                       false,
+                                       nullptr,
+                                       &actualKeys,
+                                       true  // ignoreArraysAlongPath
+    );
+
+    BSONObjSet expectedKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
+    expectedKeys.insert(makeHashKey(BSON("" << BSONNULL).firstElement()));
+    ASSERT(assertKeysetsEqual(expectedKeys, actualKeys));
+}
+
+TEST(HashKeyGeneratorTest, ArrayAtTerminalPathAlwaysFails) {
+    BSONObj obj = fromjson("{a : {b: {c: [1]}}}");
+    BSONObjSet actualKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
+    ASSERT_THROWS_CODE(ExpressionKeysPrivate::getHashKeys(obj,
+                                                          "a.b.c",
+                                                          kHashSeed,
+                                                          kHashVersion,
+                                                          true,  // isSparse
+                                                          nullptr,
+                                                          &actualKeys,
+                                                          true  // ignoreArraysAlongPath
+                                                          ),
+                       DBException,
+                       16766);
+}
+
 }  // namespace
diff --git a/src/mongo/db/index/haystack_access_method.cpp b/src/mongo/db/index/haystack_access_method.cpp
index bbfa78b1b35..b65fed348f8 100644
--- a/src/mongo/db/index/haystack_access_method.cpp
+++ b/src/mongo/db/index/haystack_access_method.cpp
@@ -65,6 +65,7 @@ HaystackAccessMethod::HaystackAccessMethod(IndexCatalogEntry* btreeState,
 }
 
 void HaystackAccessMethod::doGetKeys(const BSONObj& obj,
+                                     GetKeysContext context,
                                      BSONObjSet* keys,
                                      BSONObjSet* multikeyMetadataKeys,
                                      MultikeyPaths* multikeyPaths) const {
diff --git a/src/mongo/db/index/haystack_access_method.h b/src/mongo/db/index/haystack_access_method.h
index 99ab21dfde4..a2e4e4de53d 100644
--- a/src/mongo/db/index/haystack_access_method.h
+++ b/src/mongo/db/index/haystack_access_method.h
@@ -77,6 +77,7 @@ private:
      * geoHaystack indexes don't support tracking path-level multikey information.
      */
     void doGetKeys(const BSONObj& obj,
+                   GetKeysContext context,
                    BSONObjSet* keys,
                    BSONObjSet* multikeyMetadataKeys,
                    MultikeyPaths* multikeyPaths) const final;
diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp
index 09b1b8e1c7b..fc31c3d501a 100644
--- a/src/mongo/db/index/index_access_method.cpp
+++ b/src/mongo/db/index/index_access_method.cpp
@@ -192,7 +192,12 @@ Status AbstractIndexAccessMethod::insert(OperationContext* opCtx,
     MultikeyPaths multikeyPaths;
 
     // Delegate to the subclass.
-    getKeys(obj, options.getKeysMode, &keys, &multikeyMetadataKeys, &multikeyPaths);
+    getKeys(obj,
+            options.getKeysMode,
+            GetKeysContext::kReadOrAddKeys,
+            &keys,
+            &multikeyMetadataKeys,
+            &multikeyPaths);
 
     return insertKeys(opCtx,
                       {keys.begin(), keys.end()},
@@ -310,7 +315,12 @@ Status AbstractIndexAccessMethod::touch(OperationContext* opCtx, const BSONObj&
     // multikey when paging a document's index entries into memory.
     BSONObjSet* multikeyMetadataKeys = nullptr;
     MultikeyPaths* multikeyPaths = nullptr;
-    getKeys(obj, GetKeysMode::kEnforceConstraints, &keys, multikeyMetadataKeys, multikeyPaths);
+    getKeys(obj,
+            GetKeysMode::kEnforceConstraints,
+            GetKeysContext::kReadOrAddKeys,
+            &keys,
+            multikeyMetadataKeys,
+            multikeyPaths);
 
     std::unique_ptr<SortedDataInterface::Cursor> cursor(_newInterface->newCursor(opCtx));
     for (const auto& key : keys) {
@@ -336,6 +346,7 @@ RecordId AbstractIndexAccessMethod::findSingle(OperationContext* opCtx,
         MultikeyPaths* multikeyPaths = nullptr;
         getKeys(requestedKey,
                 GetKeysMode::kEnforceConstraints,
+                GetKeysContext::kReadOrAddKeys,
                 &keys,
                 multikeyMetadataKeys,
                 multikeyPaths);
@@ -432,12 +443,14 @@ void AbstractIndexAccessMethod::prepareUpdate(OperationContext* opCtx,
         // There's no need to compute the prefixes of the indexed fields that possibly caused the
         // index to be multikey when the old version of the document was written since the index
         // metadata isn't updated when keys are deleted.
-        getKeys(from, getKeysMode, &ticket->oldKeys, nullptr, nullptr);
+        getKeys(
+            from, getKeysMode, GetKeysContext::kRemovingKeys, &ticket->oldKeys, nullptr, nullptr);
     }
 
     if (!indexFilter || indexFilter->matchesBSON(to)) {
         getKeys(to,
                 options.getKeysMode,
+                GetKeysContext::kReadOrAddKeys,
                 &ticket->newKeys,
                 &ticket->newMultikeyMetadataKeys,
                 &ticket->newMultikeyPaths);
@@ -578,7 +591,12 @@ Status AbstractIndexAccessMethod::BulkBuilderImpl::insert(OperationContext* opCt
     MultikeyPaths multikeyPaths;
 
     try {
-        _real->getKeys(obj, options.getKeysMode, &keys, &_multikeyMetadataKeys, &multikeyPaths);
+        _real->getKeys(obj,
+                       options.getKeysMode,
+                       GetKeysContext::kReadOrAddKeys,
+                       &keys,
+                       &_multikeyMetadataKeys,
+                       &multikeyPaths);
     } catch (...) {
         return exceptionToStatus();
     }
@@ -738,6 +756,7 @@ void AbstractIndexAccessMethod::setIndexIsMultikey(OperationContext* opCtx, Mult
 
 void AbstractIndexAccessMethod::getKeys(const BSONObj& obj,
                                         GetKeysMode mode,
+                                        GetKeysContext context,
                                         BSONObjSet* keys,
                                         BSONObjSet* multikeyMetadataKeys,
                                         MultikeyPaths* multikeyPaths) const {
@@ -767,7 +786,7 @@ void AbstractIndexAccessMethod::getKeys(const BSONObj& obj,
                                               13026,
                                               13027};
     try {
-        doGetKeys(obj, keys, multikeyMetadataKeys, multikeyPaths);
+        doGetKeys(obj, context, keys, multikeyMetadataKeys, multikeyPaths);
     } catch (const AssertionException& ex) {
         // Suppress all indexing errors when mode is kRelaxConstraints.
         if (mode == GetKeysMode::kEnforceConstraints) {
diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h
index 91b35a95ddb..d7d96588d18 100644
--- a/src/mongo/db/index/index_access_method.h
+++ b/src/mongo/db/index/index_access_method.h
@@ -291,6 +291,12 @@ public:
     };
 
     /**
+     * Specifies whether getKeys is being used in the context of creating new keys or deleting
+     * existing keys.
+     */
+    enum class GetKeysContext { kRemovingKeys, kReadOrAddKeys };
+
+    /**
      * Fills 'keys' with the keys that should be generated for 'obj' on this index.
      * Based on 'mode', it will honor or ignore index constraints, e.g. duplicated key, key too
      * long, and geo index parsing errors. The ignoring of constraints is for replication due to
@@ -309,6 +315,7 @@ public:
      */
     virtual void getKeys(const BSONObj& obj,
                          GetKeysMode mode,
+                         GetKeysContext context,
                          BSONObjSet* keys,
                          BSONObjSet* multikeyMetadataKeys,
                          MultikeyPaths* multikeyPaths) const = 0;
@@ -520,6 +527,7 @@ public:
 
     void getKeys(const BSONObj& obj,
                  GetKeysMode mode,
+                 GetKeysContext context,
                  BSONObjSet* keys,
                  BSONObjSet* multikeyMetadataKeys,
                  MultikeyPaths* multikeyPaths) const final;
@@ -546,6 +554,7 @@ protected:
      * information that must be stored in a reserved keyspace within the index.
      */
     virtual void doGetKeys(const BSONObj& obj,
+                           GetKeysContext context,
                            BSONObjSet* keys,
                            BSONObjSet* multikeyMetadataKeys,
                            MultikeyPaths* multikeyPaths) const = 0;
diff --git a/src/mongo/db/index/s2_access_method.cpp b/src/mongo/db/index/s2_access_method.cpp
index 615838142c8..14192d4d185 100644
--- a/src/mongo/db/index/s2_access_method.cpp
+++ b/src/mongo/db/index/s2_access_method.cpp
@@ -124,6 +124,7 @@ StatusWith<BSONObj> S2AccessMethod::fixSpec(const BSONObj& specObj) {
 }
 
 void S2AccessMethod::doGetKeys(const BSONObj& obj,
+                               GetKeysContext context,
                                BSONObjSet* keys,
                                BSONObjSet* multikeyMetadataKeys,
                                MultikeyPaths* multikeyPaths) const {
diff --git a/src/mongo/db/index/s2_access_method.h b/src/mongo/db/index/s2_access_method.h
index 164b38c5af5..c1496d1af58 100644
--- a/src/mongo/db/index/s2_access_method.h
+++ b/src/mongo/db/index/s2_access_method.h
@@ -65,6 +65,7 @@ private:
      * be multikey as a result of inserting 'keys'.
      */
     void doGetKeys(const BSONObj& obj,
+                   GetKeysContext context,
                    BSONObjSet* keys,
                    BSONObjSet* multikeyMetadataKeys,
                    MultikeyPaths* multikeyPaths) const final;
diff --git a/src/mongo/db/index/wildcard_access_method.cpp b/src/mongo/db/index/wildcard_access_method.cpp
index cfa6adc1e46..07e551c3b11 100644
--- a/src/mongo/db/index/wildcard_access_method.cpp
+++ b/src/mongo/db/index/wildcard_access_method.cpp
@@ -51,6 +51,7 @@ bool WildcardAccessMethod::shouldMarkIndexAsMultikey(
 }
 
 void WildcardAccessMethod::doGetKeys(const BSONObj& obj,
+                                     GetKeysContext context,
                                      BSONObjSet* keys,
                                      BSONObjSet* multikeyMetadataKeys,
                                      MultikeyPaths* multikeyPaths) const {
diff --git a/src/mongo/db/index/wildcard_access_method.h b/src/mongo/db/index/wildcard_access_method.h
index 45d866c1020..0480c14c3a6 100644
--- a/src/mongo/db/index/wildcard_access_method.h
+++ b/src/mongo/db/index/wildcard_access_method.h
@@ -93,6 +93,7 @@ public:
 
 private:
     void doGetKeys(const BSONObj& obj,
+                   GetKeysContext context,
                    BSONObjSet* keys,
                    BSONObjSet* multikeyMetadataKeys,
                    MultikeyPaths* multikeyPaths) const final;
diff --git a/src/mongo/dbtests/validate_tests.cpp b/src/mongo/dbtests/validate_tests.cpp
index c6e46ba2ce5..c5054aeff2a 100644
--- a/src/mongo/dbtests/validate_tests.cpp
+++ b/src/mongo/dbtests/validate_tests.cpp
@@ -832,6 +832,7 @@ public:
             BSONObjSet keys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
             iam->getKeys(actualKey,
                          IndexAccessMethod::GetKeysMode::kRelaxConstraintsUnfiltered,
+                         IndexAccessMethod::GetKeysContext::kReadOrAddKeys,
                          &keys,
                          nullptr,
                          nullptr);
@@ -1258,6 +1259,7 @@ public:
             BSONObjSet keys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
             iam->getKeys(actualKey,
                          IndexAccessMethod::GetKeysMode::kRelaxConstraintsUnfiltered,
+                         IndexAccessMethod::GetKeysContext::kReadOrAddKeys,
                          &keys,
                          nullptr,
                          nullptr);
@@ -1449,6 +1451,7 @@ public:
             BSONObjSet keys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
             iam->getKeys(actualKey,
                          IndexAccessMethod::GetKeysMode::kRelaxConstraintsUnfiltered,
+                         IndexAccessMethod::GetKeysContext::kReadOrAddKeys,
                          &keys,
                          nullptr,
                          nullptr);
@@ -1482,6 +1485,7 @@ public:
             BSONObjSet keys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
             iam->getKeys(actualKey,
                          IndexAccessMethod::GetKeysMode::kRelaxConstraintsUnfiltered,
+                         IndexAccessMethod::GetKeysContext::kReadOrAddKeys,
                          &keys,
                          nullptr,
                          nullptr);