SERVER-57588 Inconsistent query results when an array position is indexed whose value is an array

(cherry picked from commit 90699509e15b33fda10832e79efcd158aee1f0eb)

4 files changed, 151 insertions, 0 deletions

diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index 8cc02dd4154..4d82cbbcac6 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -70,6 +70,8 @@ last-continuous:
       test_file: jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js
     - ticket: SERVER-59721
       test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js
+    - ticket: SERVER-57588
+      test_file: jstests/core/match_numeric_components.js
     - ticket: SERVER-60682
       test_file: jstests/sharding/coordinate_txn_commit_with_tickets_exhausted.js
     - ticket: SERVER-60685
@@ -296,6 +298,8 @@ last-lts:
       test_file: jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js
     - ticket: SERVER-59721
       test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js
+    - ticket: SERVER-57588
+      test_file: jstests/core/match_numeric_components.js
     - ticket: SERVER-60682
       test_file: jstests/sharding/coordinate_txn_commit_with_tickets_exhausted.js
     - ticket: SERVER-60685
diff --git a/jstests/core/match_numeric_components.js b/jstests/core/match_numeric_components.js
index 07a58507b8e..7f20b3710ab 100644
--- a/jstests/core/match_numeric_components.js
+++ b/jstests/core/match_numeric_components.js
@@ -350,4 +350,100 @@ assert.commandWorked(coll2.insert(kRegexDocs));
     const expected = [{_id: 1, "b": "hello"}];
     assert.sameMembers(res, expected);
 }
+
+// The tests below indirectly make sure that an index scan is not chosen when $elemMatch is
+// against a indexed positional path component because it is not possible to generate index
+// bounds from the $elemMatch conditions. If an index scan is chosen, then the corresponding
+// queries would produce a wrong result.
+// More precisely for an index like {"a.0": 1} and a document {a: [[1, 2, 3]]}, the nested array is
+// not unwound during index key generation. That is, there is a single index key {"": [1, 2, 3]}
+// rather than three separate index keys, {"": 1}, {"": 2}, {"": 3}. This precludes the ability to
+// generate index bounds for $elemMatch predicates on "a.0" because "a.0" refers to the whole array
+// [1, 2, 3], and not its individual members.
+{
+    // Test with $in.
+    assert(coll.drop());
+    const expectedDoc = {_id: 42, "a": [["b"], ["c"]]};
+    assert.commandWorked(coll.insert(expectedDoc));
+    const query = {"a.0": {$elemMatch: {$in: ["b"]}}};
+
+    const res1 = coll.find(query).toArray();
+    assert.commandWorked(coll.createIndex({"a.0": 1}));
+    const res2 = coll.find(query).toArray();
+    assert.sameMembers([expectedDoc], res1);
+    assert.sameMembers([expectedDoc], res2);
+}
+
+// Tests with equality. Add some data for the next few tests.
+coll.drop();
+assert.commandWorked(coll.insert({_id: 0, f0: 'zz', f1: 5}));
+assert.commandWorked(coll.insert({_id: 1, f0: 'zz', f1: [3, 5]}));
+assert.commandWorked(coll.insert({_id: 4, f0: 'zz', f1: [3, 5, [7, 9]]}));
+assert.commandWorked(coll.insert({_id: 2, f0: 'zz', f1: [[3, 5], [5, 7]]}));
+assert.commandWorked(coll.insert({_id: 3, f0: 'zz', f1: [[[0], [3, 5]], [[0], [5, 7]]]}));
+
+{
+    const res1 = coll.find({'f1.0': {$elemMatch: {$eq: 5}}}).toArray();
+    const res2 = coll.find({'f1.0': {$elemMatch: {$eq: [3, 5]}}}).toArray();
+    assert.commandWorked(coll.createIndex({'f1.0': 1}));
+    const res3 = coll.find({'f1.0': {$elemMatch: {$eq: 5}}}).toArray();
+    const res4 = coll.find({'f1.0': {$elemMatch: {$eq: [3, 5]}}}).toArray();
+    const expected1 = [{_id: 2, f0: 'zz', f1: [[3, 5], [5, 7]]}];
+    const expected2 = [{_id: 3, f0: 'zz', f1: [[[0], [3, 5]], [[0], [5, 7]]]}];
+    assert.sameMembers(expected1, res1);
+    assert.sameMembers(expected1, res3);
+    assert.sameMembers(expected2, res2);
+    assert.sameMembers(expected2, res4);
+    assert.commandWorked(coll.dropIndex({'f1.0': 1}));
+}
+
+{
+    // Compound index.
+    const res1 = coll.find({'f0': 'zz', 'f1.0': {$elemMatch: {$eq: 5}}}).toArray();
+    assert.commandWorked(coll.createIndex({'f0': 1, 'f1.0': 1}));
+    const res2 = coll.find({'f0': 'zz', 'f1.0': {$elemMatch: {$eq: 5}}}).toArray();
+    const expected = [{_id: 2, f0: 'zz', f1: [[3, 5], [5, 7]]}];
+    assert.sameMembers(expected, res1);
+    assert.sameMembers(expected, res2);
+    assert.commandWorked(coll.dropIndex({'f0': 1, 'f1.0': 1}));
+}
+
+{
+    // Two-levels of array nesting.
+    const res1 = coll.find({'f1.0.1': {$elemMatch: {$eq: 3}}}).toArray();
+    assert.commandWorked(coll.createIndex({'f1.0.1': 1}));
+    const res2 = coll.find({'f1.0.1': {$elemMatch: {$eq: 3}}}).toArray();
+    const expected = [{_id: 3, f0: 'zz', f1: [[[0], [3, 5]], [[0], [5, 7]]]}];
+    assert.sameMembers(expected, res1);
+    assert.sameMembers(expected, res2);
+}
+
+{
+    assert(coll.drop());
+    assert.commandWorked(coll.createIndex({'f1.0': 1}));
+    assert.commandWorked(coll.insert({_id: 1, f1: [[42, 5], [77, 99]]}));
+
+    const res1 = coll.find({'f1.0': {$elemMatch: {$eq: 5}}}).toArray();
+    assert.sameMembers([{_id: 1, f1: [[42, 5], [77, 99]]}], res1);
+
+    // Object with numeric field component, and no nested arrays.
+    assert.commandWorked(coll.insert({_id: 2, f1: {0: [42, 5], 1: [77, 99]}}));
+    const res2 = coll.find({'f1.0': {$elemMatch: {$eq: 5}}}).toArray();
+    assert.sameMembers(
+        [{_id: 1, f1: [[42, 5], [77, 99]]}, {_id: 2, f1: {'0': [42, 5], '1': [77, 99]}}], res2);
+}
+
+{
+    assert(coll.drop());
+    assert.commandWorked(coll.createIndex({'0': 1}));
+
+    assert.commandWorked(coll.insert({_id: 1, '0': [42, 5]}));
+    const res1 = coll.find({'0': {$elemMatch: {$eq: 5}}}).toArray();
+    assert.sameMembers([{'0': [42, 5], _id: 1}], res1);
+
+    assert.commandWorked(coll.createIndex({'0.1': 1}));
+    assert.commandWorked(coll.insert({_id: 2, '0': {0: [42], 1: [5]}}));
+    const res2 = coll.find({'0.1': {$elemMatch: {$eq: 5}}}).toArray();
+    assert.sameMembers([{'0': {'0': [42], '1': [5]}, _id: 2}], res2);
+}
 })();
diff --git a/src/mongo/db/query/planner_ixselect.cpp b/src/mongo/db/query/planner_ixselect.cpp
index 83ec4b99fe9..ff565bbae56 100644
--- a/src/mongo/db/query/planner_ixselect.cpp
+++ b/src/mongo/db/query/planner_ixselect.cpp
@@ -475,6 +475,16 @@ bool QueryPlannerIXSelect::_compatible(const BSONElement& keyPatternElt,
             newContext.fullPathToParentElemMatch = fullPathToNode;
             newContext.innermostParentElemMatch = static_cast<ElemMatchValueMatchExpression*>(node);
 
+            FieldRef path(fullPathToNode);
+            // If the index path has at least two components, and the last component of the path is
+            // numeric, this component could be an array index because the preceding path component
+            // may contain an array. Currently it is not known whether the preceding path component
+            // could be an array because indexes which positionally index array elements are not
+            // considered multikey.
+            if (path.numParts() > 1 && path.isNumericPathComponentStrict(path.numParts() - 1)) {
+                return false;
+            }
+
             auto&& children = node->getChildVector();
             if (!std::all_of(children->begin(), children->end(), [&](auto&& child) {
                     const auto newPath = fullPathToNode.toString() + child->path();
diff --git a/src/mongo/db/query/query_planner_array_test.cpp b/src/mongo/db/query/query_planner_array_test.cpp
index 1797ff56265..19dfc99dbcd 100644
--- a/src/mongo/db/query/query_planner_array_test.cpp
+++ b/src/mongo/db/query/query_planner_array_test.cpp
@@ -2582,6 +2582,47 @@ TEST_F(QueryPlannerTest,
         "}}}}}}}");
 }
 
+// Test for older versions of indexes where it is possible to have empty MultikeyPaths,
+// but still the index is considered multikey. In that case the index should not be
+// considered to evaluate an $elemMatch predicate with a positional path component.
+TEST_F(QueryPlannerTest, ElemMatchValueMultikeyIndexEmptyMultikeyPaths) {
+    addIndex(BSON("a.0" << 1), true);
+    runQuery(fromjson("{'a.0': {$elemMatch: {$eq: 42}}}"));
+
+    assertNumSolutions(1U);
+    assertSolutionExists("{cscan: {dir: 1}}");
+}
+
+TEST_F(QueryPlannerTest, ElemMatchValuePositionalIndexPath1) {
+    MultikeyPaths multikeyPaths{MultikeyComponents{}};
+    addIndex(BSON("f1.0" << 1), multikeyPaths);
+    runQuery(fromjson("{'f1.0': {$elemMatch: {$eq: 42}}}"));
+
+    assertNumSolutions(1U);
+    assertSolutionExists("{cscan: {dir: 1}}");
+}
+
+TEST_F(QueryPlannerTest, ElemMatchValuePositionalIndexPath2) {
+    MultikeyPaths multikeyPaths{{0U}};
+    addIndex(BSON("f1.0" << 1), multikeyPaths);
+    runQuery(fromjson("{'f1.0': {$elemMatch: {$eq: 42}}}"));
+
+    assertNumSolutions(1U);
+    assertSolutionExists("{cscan: {dir: 1}}");
+}
+
+TEST_F(QueryPlannerTest, ElemMatchValuePositionalIndexPath3) {
+    MultikeyPaths multikeyPaths{MultikeyComponents{}};
+    addIndex(BSON("0" << 1), multikeyPaths);
+    runQuery(fromjson("{'0': {$elemMatch: {$eq: 42}}}"));
+
+    assertNumSolutions(2U);
+    assertSolutionExists("{cscan: {dir: 1}}");
+    assertSolutionExists(
+        "{fetch: {node: {ixscan: {pattern: {'0': 1}, bounds: "
+        "{'0': [[42,42,true,true]]}}}}}");
+}
+
 TEST_F(QueryPlannerTest, CompoundIndexBoundsDottedNotEqualsNullMultiKey) {
     const bool isMultiKey = true;
     addIndex(BSON("a.b" << 1 << "c.d" << 1), isMultiKey);