SERVER-36362 Do not consider an 'allPaths' index for a $text query

4 files changed, 175 insertions, 6 deletions

diff --git a/jstests/noPassthroughWithMongod/all_paths_and_text_indexes.js b/jstests/noPassthroughWithMongod/all_paths_and_text_indexes.js
new file mode 100644
index 00000000000..2b515f89d59
--- /dev/null
+++ b/jstests/noPassthroughWithMongod/all_paths_and_text_indexes.js
@@ -0,0 +1,101 @@
+/**
+ * Tests that a {$**: 1} index can coexist with a {$**: 'text'} index in the same collection.
+ *
+ * Tagged as 'assumes_unsharded_collection' so that the expected relationship between the number of
+ * IXSCANs in the explain output and the number of fields in the indexed documents is not distorted
+ * by being spread across multiple shards.
+ *
+ * @tags: [assumes_unsharded_collection]
+ *
+ * TODO: SERVER-36198: Move this test back to jstests/core/
+ */
+(function() {
+    "use strict";
+
+    load("jstests/aggregation/extras/utils.js");  // For arrayEq.
+    load("jstests/libs/analyze_plan.js");         // For getPlanStages and planHasStage.
+
+    const assertArrayEq = (l, r) => assert(arrayEq(l, r), tojson(l) + " != " + tojson(r));
+
+    const coll = db.all_paths_and_text_indexes;
+    coll.drop();
+
+    // Runs a single allPaths query test, confirming that an indexed solution exists, that the $**
+    // index on the given 'expectedPath' was used to answer the query, and that the results are
+    // identical to those obtained via COLLSCAN.
+    function assertAllPathsQuery(query, expectedPath) {
+        // Explain the query, and determine whether an indexed solution is available.
+        const explainOutput = coll.find(query).explain("executionStats");
+        const ixScans = getPlanStages(explainOutput.queryPlanner.winningPlan, "IXSCAN");
+        // Verify that the winning plan uses the $** index with the expected path.
+        assert.eq(ixScans.length, 1);
+        assert.docEq(ixScans[0].keyPattern, {"$_path": 1, [expectedPath]: 1});
+        // Verify that the results obtained from the $** index are identical to a COLLSCAN.
+        assertArrayEq(coll.find(query).toArray(), coll.find(query).hint({$natural: 1}).toArray());
+    }
+
+    // Insert documents containing the field '_fts', which is reserved when using a $text index.
+    assert.commandWorked(coll.insert({_id: 1, a: 1, _fts: 1, textToSearch: "banana"}));
+    assert.commandWorked(coll.insert({_id: 2, a: 1, _fts: 2, textToSearch: "bananas"}));
+    assert.commandWorked(coll.insert({_id: 3, a: 1, _fts: 3}));
+
+    // Required in order to build $** indexes.
+    assert.commandWorked(
+        db.adminCommand({setParameter: 1, internalQueryAllowAllPathsIndexes: true}));
+    try {
+        // Build an allPaths index, and verify that it can be used to query for the field '_fts'.
+        assert.commandWorked(coll.createIndex({"$**": 1}));
+        assertAllPathsQuery({_fts: {$gt: 0, $lt: 4}}, '_fts');
+
+        // Perform the tests below for simple and compound $text indexes.
+        for (let textIndex of[{'$**': 'text'}, {a: 1, '$**': 'text'}]) {
+            // Build the appropriate text index.
+            assert.commandWorked(coll.createIndex(textIndex, {name: "textIndex"}));
+
+            // Confirm that the $** index can still be used to query for the '_fts' field outside of
+            // a $text query.
+            assertAllPathsQuery({_fts: {$gt: 0, $lt: 4}}, '_fts');
+
+            // Confirm that $** does not generate a candidate plan for $text search, including cases
+            // when the query filter contains a compound field in the $text index.
+            const textQuery =
+                Object.assign(textIndex.a ? {a: 1} : {}, {$text: {$search: 'banana'}});
+            let explainOut = assert.commandWorked(coll.find(textQuery).explain("executionStats"));
+            assert(planHasStage(coll.getDB(), explainOut.queryPlanner.winningPlan, "TEXT"));
+            assert.eq(explainOut.queryPlanner.rejectedPlans.length, 0);
+            assert.eq(explainOut.executionStats.nReturned, 2);
+
+            // Confirm that $** does not generate a candidate plan for $text search, including cases
+            // where the query filter contains a field which is not present in the text index.
+            explainOut =
+                assert.commandWorked(coll.find(Object.assign({_fts: {$gt: 0, $lt: 4}}, textQuery))
+                                         .explain("executionStats"));
+            assert(planHasStage(coll.getDB(), explainOut.queryPlanner.winningPlan, "TEXT"));
+            assert.eq(explainOut.queryPlanner.rejectedPlans.length, 0);
+            assert.eq(explainOut.executionStats.nReturned, 2);
+
+            // Confirm that the $** index can be used alongside a $text predicate in an $or.
+            explainOut = assert.commandWorked(
+                coll.find({$or: [{_fts: 3}, textQuery]}).explain("executionStats"));
+            assert.eq(explainOut.queryPlanner.rejectedPlans.length, 0);
+            assert.eq(explainOut.executionStats.nReturned, 3);
+
+            const textOrAllPaths = getPlanStages(explainOut.queryPlanner.winningPlan, "OR").shift();
+            assert.eq(textOrAllPaths.inputStages.length, 2);
+            const textBranch = (textOrAllPaths.inputStages[0].stage === "TEXT" ? 0 : 1);
+            const allPathsBranch = (textBranch + 1) % 2;
+            assert.eq(textOrAllPaths.inputStages[textBranch].stage, "TEXT");
+            assert.eq(textOrAllPaths.inputStages[allPathsBranch].stage, "IXSCAN");
+            assert.eq(textOrAllPaths.inputStages[allPathsBranch].keyPattern, {$_path: 1, _fts: 1});
+
+            // Drop the index so that a different text index can be created.
+            assert.commandWorked(coll.dropIndex("textIndex"));
+        }
+    } finally {
+        // Disable $** indexes once the tests have either completed or failed.
+        assert.commandWorked(
+            db.adminCommand({setParameter: 1, internalQueryAllowAllPathsIndexes: false}));
+        // TODO: SERVER-36444 remove this coll.drop because validation should work now.
+        coll.drop();
+    }
+})();
diff --git a/src/mongo/db/query/planner_ixselect.cpp b/src/mongo/db/query/planner_ixselect.cpp
index eb9cbfbb55f..70deacf2d5b 100644
--- a/src/mongo/db/query/planner_ixselect.cpp
+++ b/src/mongo/db/query/planner_ixselect.cpp
@@ -686,6 +686,7 @@ void QueryPlannerIXSelect::_rateIndices(MatchExpression* node,
 // static
 void QueryPlannerIXSelect::stripInvalidAssignments(MatchExpression* node,
                                                    const vector<IndexEntry>& indices) {
+    stripInvalidAssignmentsToAllPathsIndexes(node, indices);
     stripInvalidAssignmentsToTextIndexes(node, indices);
 
     if (MatchExpression::GEO != node->matchType() &&
@@ -874,6 +875,34 @@ void QueryPlannerIXSelect::stripInvalidAssignmentsToPartialIndices(
 }
 
 //
+// AllPaths index invalid assignments.
+//
+void QueryPlannerIXSelect::stripInvalidAssignmentsToAllPathsIndexes(
+    MatchExpression* root, const vector<IndexEntry>& indices) {
+    for (size_t idx = 0; idx < indices.size(); ++idx) {
+        // Skip over all indexes except $**.
+        if (indices[idx].type != IndexType::INDEX_ALLPATHS) {
+            continue;
+        }
+        // If we have a $** index, check whether we have a TEXT node in the MatchExpression tree.
+        const std::function<MatchExpression*(MatchExpression*)> findTextNode = [&](auto* node) {
+            if (node->matchType() == MatchExpression::TEXT) {
+                return node;
+            }
+            for (size_t i = 0; i < node->numChildren(); ++i) {
+                if (auto* foundNode = findTextNode(node->getChild(i)))
+                    return foundNode;
+            }
+            return static_cast<MatchExpression*>(nullptr);
+        };
+        // If so, remove the $** index from the node's relevant tags.
+        if (auto* textNode = findTextNode(root)) {
+            removeIndexRelevantTag(textNode, idx);
+        }
+    }
+}
+
+//
 // Text index quirks
 //
 
diff --git a/src/mongo/db/query/planner_ixselect.h b/src/mongo/db/query/planner_ixselect.h
index b1834a1610e..b0181fab150 100644
--- a/src/mongo/db/query/planner_ixselect.h
+++ b/src/mongo/db/query/planner_ixselect.h
@@ -228,6 +228,17 @@ private:
                                                          const std::vector<IndexEntry>& indices);
 
     /**
+     * This function strips RelevantTag assignments to expanded 'allPaths' indexes, in cases where
+     * the assignment is incompatible with the query.
+     *
+     * Specifically, if the query has a TEXT node with both 'text' and 'allPaths' indexes present,
+     * then the 'allPaths' index will mark itself as relevant to the '_fts' path reported by the
+     * TEXT node. We therefore remove any such misassigned 'allPaths' tags here.
+     */
+    static void stripInvalidAssignmentsToAllPathsIndexes(MatchExpression* root,
+                                                         const std::vector<IndexEntry>& indices);
+
+    /**
      * This function strips RelevantTag assignments to partial indices, where the assignment is
      * incompatible with the index's filter expression.
      *
diff --git a/src/mongo/db/query/query_planner_all_paths_index_test.cpp b/src/mongo/db/query/query_planner_all_paths_index_test.cpp
index 0e6374252c1..c758a0c48a8 100644
--- a/src/mongo/db/query/query_planner_all_paths_index_test.cpp
+++ b/src/mongo/db/query/query_planner_all_paths_index_test.cpp
@@ -585,12 +585,6 @@ TEST_F(QueryPlannerAllPathsTest, InBasicOrEquivalent) {
         "bounds: {'$_path': [['a','a',true,true]], a: [[1,1,true,true],[2,2,true,true]]}}}}}");
 }
 
-// TODO SERVER-35335: Add testing for Min/Max.
-// TODO SERVER-36517: Add testing for DISTINCT_SCAN.
-// TODO SERVER-35336: Add testing for partialFilterExpression.
-// TODO SERVER-35331: Add testing for hints.
-// TODO SERVER-36145: Add testing for non-blocking sort.
-
 //
 // Index intersection tests.
 //
@@ -668,4 +662,38 @@ TEST_F(QueryPlannerAllPathsTest, AllPathsIndexDoesNotParticipateInIndexIntersect
         "{fetch: {filter:{b:{$gt:10}}, node: {ixscan: {filter: null, pattern: {$_path:1, a:1}}}}}");
 }
 
+//
+// AllPaths and $text index tests.
+//
+
+TEST_F(QueryPlannerAllPathsTest, AllPathsIndexDoesNotSupplyCandidatePlanForTextSearch) {
+    addAllPathsIndex(BSON("$**" << 1));
+    addIndex(BSON("a" << 1 << "_fts"
+                      << "text"
+                      << "_ftsx"
+                      << 1));
+
+    // Confirm that the allPaths index generates candidate plans for queries which do not include a
+    // $text predicate.
+    runQuery(fromjson("{a: 10, b: 10}"));
+    ASSERT_EQUALS(getNumSolutions(), 2U);
+    assertSolutionExists(
+        "{fetch: {filter: {b: 10}, node: {ixscan: {filter: null, pattern: {'$_path': 1, a: 1}}}}}");
+    assertSolutionExists(
+        "{fetch: {filter: {a: 10}, node: {ixscan: {filter: null, pattern: {'$_path': 1, b: 1}}}}}");
+
+    // Confirm that the allPaths index does not produce any candidate plans when a query includes a
+    // $text predicate, even for non-$text predicates which may be present in the query.
+    runQuery(fromjson("{a: 10, b: 10, $text: {$search: 'banana'}}"));
+    ASSERT_EQUALS(getNumSolutions(), 1U);
+    assertSolutionExists(
+        "{fetch: {filter: {b: 10}, node: {text: {prefix: {a: 10}, search: 'banana'}}}}");
+}
+
+// TODO SERVER-35335: Add testing for Min/Max.
+// TODO SERVER-36517: Add testing for DISTINCT_SCAN.
+// TODO SERVER-35336: Add testing for partialFilterExpression.
+// TODO SERVER-35331: Add testing for hints.
+// TODO SERVER-36145: Add testing for non-blocking sort.
+
 }  // namespace mongo