diff options
author | Ian Boros <ian.boros@10gen.com> | 2018-09-06 16:39:53 -0400 |
---|---|---|
committer | Ian Boros <ian.boros@10gen.com> | 2018-09-20 16:26:20 -0400 |
commit | 5ed54bec92fc15009877e2d579fe59785bffdde7 (patch) | |
tree | f190e99159e98dde1d5f0d13f457c4ec2c1b12f1 | |
parent | 64672e68a25e1c4c53a9e1e974036b02fdda2cc5 (diff) | |
download | mongo-5ed54bec92fc15009877e2d579fe59785bffdde7.tar.gz |
SERVER-36731 Ban object inequality and $in with unsupported values when using allPaths indexes
-rw-r--r-- | jstests/noPassthroughWithMongod/all_paths_basic_index_bounds.js | 12 | ||||
-rw-r--r-- | src/mongo/db/query/plan_cache_indexability.cpp | 61 | ||||
-rw-r--r-- | src/mongo/db/query/planner_ixselect.cpp | 93 | ||||
-rw-r--r-- | src/mongo/db/query/planner_ixselect.h | 11 | ||||
-rw-r--r-- | src/mongo/db/query/planner_ixselect_test.cpp | 43 | ||||
-rw-r--r-- | src/mongo/db/query/query_planner_all_paths_index_test.cpp | 91 | ||||
-rw-r--r-- | src/mongo/db/query/query_planner_test.cpp | 33 |
7 files changed, 277 insertions, 67 deletions
diff --git a/jstests/noPassthroughWithMongod/all_paths_basic_index_bounds.js b/jstests/noPassthroughWithMongod/all_paths_basic_index_bounds.js index 8252fa5f6bf..4934917fac4 100644 --- a/jstests/noPassthroughWithMongod/all_paths_basic_index_bounds.js +++ b/jstests/noPassthroughWithMongod/all_paths_basic_index_bounds.js @@ -64,6 +64,12 @@ {expression: {$exists: false}, bounds: null}, {expression: {$eq: null}, bounds: null}, {expression: {$ne: null}, bounds: null}, + {expression: {$eq: {abc: 1}}, bounds: null}, + {expression: {$lt: {abc: 1}}, bounds: null}, + {expression: {$ne: {abc: 1}}, bounds: null}, + {expression: {$lt: {abc: 1}, $gt: {abc: 1}}, bounds: null}, + {expression: {$in: [{abc: 1}, 1, 2, 3]}, bounds: null}, + {expression: {$in: [null, 1, 2, 3]}, bounds: null}, {expression: {$ne: null, $exists: true}, bounds: ['[MinKey, MaxKey]'], subpathBounds: true}, // In principle we could have tighter bounds for this. See SERVER-36765. {expression: {$eq: null, $exists: true}, bounds: ['[MinKey, MaxKey]'], subpathBounds: true}, @@ -110,7 +116,11 @@ // and projection, or if the current operation is not supported by $** indexes, // confirm that no indexed solution was found. if (!expectedPaths.includes(path) || op.bounds === null) { - assert.eq(ixScans.length, 0); + assert.eq(ixScans.length, + 0, + () => "Bounds check for operation: " + tojson(op) + + " failed. Expected no IXSCAN plans to be generated, but got " + + tojson(ixScans)); continue; } diff --git a/src/mongo/db/query/plan_cache_indexability.cpp b/src/mongo/db/query/plan_cache_indexability.cpp index c541a2fe4af..860381f8b93 100644 --- a/src/mongo/db/query/plan_cache_indexability.cpp +++ b/src/mongo/db/query/plan_cache_indexability.cpp @@ -40,6 +40,7 @@ #include "mongo/db/query/collation/collation_index_key.h" #include "mongo/db/query/collation/collator_interface.h" #include "mongo/db/query/index_entry.h" +#include "mongo/db/query/planner_ixselect.h" #include "mongo/stdx/memory.h" #include <memory> @@ -47,49 +48,6 @@ namespace mongo { namespace { -bool canUseAllPathsIndex(BSONElement elt, MatchExpression::MatchType matchType) { - if (elt.type() == BSONType::Object) { - return false; - } - - if (elt.type() == BSONType::Array) { - // We only support equality to empty array. - return elt.embeddedObject().isEmpty() && matchType == MatchExpression::EQ; - } - - return true; -} - -bool supportedByAllPathsIndex(const MatchExpression* queryExpr) { - if (ComparisonMatchExpression::isComparisonMatchExpression(queryExpr)) { - const ComparisonMatchExpression* cmpExpr = - static_cast<const ComparisonMatchExpression*>(queryExpr); - - return canUseAllPathsIndex(cmpExpr->getData(), cmpExpr->matchType()); - } else if (queryExpr->matchType() == MatchExpression::MATCH_IN) { - const auto* queryExprIn = static_cast<const InMatchExpression*>(queryExpr); - - return std::all_of( - queryExprIn->getEqualities().begin(), - queryExprIn->getEqualities().end(), - [](const BSONElement& elt) { return canUseAllPathsIndex(elt, MatchExpression::EQ); }); - } - - return true; -}; - -bool supportedBySparseIndex(const MatchExpression* queryExpr) { - if (queryExpr->matchType() == MatchExpression::EQ) { - const auto* queryExprEquality = static_cast<const EqualityMatchExpression*>(queryExpr); - return !queryExprEquality->getData().isNull(); - } else if (queryExpr->matchType() == MatchExpression::MATCH_IN) { - const auto* queryExprIn = static_cast<const InMatchExpression*>(queryExpr); - return !queryExprIn->hasNull(); - } else { - return true; - } -}; - IndexabilityDiscriminator getPartialIndexDiscriminator(const MatchExpression* filterExpr) { return [filterExpr](const MatchExpression* queryExpr) { return expression::isSubsetOf(queryExpr, filterExpr); @@ -124,13 +82,21 @@ IndexabilityDiscriminator getCollatedIndexDiscriminator(const CollatorInterface* return true; }; } + +bool nodeIsConservativelySupportedBySparseIndex(const MatchExpression* me) { + // When an expression is in an $elemMatch, a sparse index may be able to support an equality to + // null. We don't track whether or not a match expression is in an $elemMatch when generating + // the plan cache key, so we make the conservative assumption that it is not. + const bool inElemMatch = false; + return QueryPlannerIXSelect::nodeIsSupportedBySparseIndex(me, inElemMatch); +} } void PlanCacheIndexabilityState::processSparseIndex(const std::string& indexName, const BSONObj& keyPattern) { for (BSONElement elem : keyPattern) { _pathDiscriminatorsMap[elem.fieldNameStringData()][indexName].addDiscriminator( - supportedBySparseIndex); + nodeIsConservativelySupportedBySparseIndex); } } @@ -187,8 +153,11 @@ IndexToDiscriminatorMap PlanCacheIndexabilityState::buildAllPathsDiscriminators( if (allPathsDiscriminator.projectionExec->applyProjectionToOneField(path)) { CompositeIndexabilityDiscriminator& cid = ret[allPathsDiscriminator.catalogName]; - cid.addDiscriminator(supportedByAllPathsIndex); - cid.addDiscriminator(supportedBySparseIndex); + // We can use these 'shallow' functions because the code building the plan cache key + // will descend the match expression for us, and check the discriminator's return value + // at each node. + cid.addDiscriminator(QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex); + cid.addDiscriminator(nodeIsConservativelySupportedBySparseIndex); cid.addDiscriminator(getCollatedIndexDiscriminator(allPathsDiscriminator.collator)); if (allPathsDiscriminator.filterExpr) { cid.addDiscriminator( diff --git a/src/mongo/db/query/planner_ixselect.cpp b/src/mongo/db/query/planner_ixselect.cpp index b6b2633a944..4cc85df1a5f 100644 --- a/src/mongo/db/query/planner_ixselect.cpp +++ b/src/mongo/db/query/planner_ixselect.cpp @@ -131,6 +131,20 @@ void expandIndex(const IndexEntry& allPathsIndex, out->push_back(std::move(entry)); } } + +bool canUseAllPathsIndex(BSONElement elt, MatchExpression::MatchType matchType) { + if (elt.type() == BSONType::Object) { + return false; + } + + if (elt.type() == BSONType::Array) { + // We only support equality to empty array. + return elt.embeddedObject().isEmpty() && matchType == MatchExpression::EQ; + } + + return true; +} + } // namespace bool QueryPlannerIXSelect::notEqualsNullCanUseIndex(const IndexEntry& index, @@ -432,26 +446,9 @@ bool QueryPlannerIXSelect::_compatible(const BSONElement& keyPatternElt, } if (indexedFieldType.empty()) { - // Can't use a sparse index for $eq with a null element, unless the equality is within a - // $elemMatch expression since the latter implies a match on the literal element 'null'. - // - // We can use a sparse index for $_internalExprEq with a null element. Expression language - // equality-to-null semantics are that only literal nulls match. Sparse indexes contain - // index keys for literal nulls, but not for missing elements. - if (exprtype == MatchExpression::EQ && index.sparse && !isChildOfElemMatchValue) { - const EqualityMatchExpression* expr = static_cast<const EqualityMatchExpression*>(node); - if (expr->getData().isNull()) { - return false; - } - } - - // Can't use a sparse index for $in with a null element, unless the $eq is within a - // $elemMatch expression since the latter implies a match on the literal element 'null'. - if (exprtype == MatchExpression::MATCH_IN && index.sparse && !isChildOfElemMatchValue) { - const InMatchExpression* expr = static_cast<const InMatchExpression*>(node); - if (expr->hasNull()) { - return false; - } + // We can't use a sparse index for certain match expressions. + if (index.sparse && !nodeIsSupportedBySparseIndex(node, isChildOfElemMatchValue)) { + return false; } // We can't use a btree-indexed field for geo expressions. @@ -520,6 +517,10 @@ bool QueryPlannerIXSelect::_compatible(const BSONElement& keyPatternElt, } } + if (index.type == IndexType::INDEX_ALLPATHS && !nodeIsSupportedByAllPathsIndex(node)) { + return false; + } + // We can only index EQ using text indices. This is an artificial limitation imposed by // FTSSpec::getIndexPrefix() which will fail if there is not an EQ predicate on each // index prefix field of the text index. @@ -625,6 +626,58 @@ bool QueryPlannerIXSelect::_compatible(const BSONElement& keyPatternElt, } } +bool QueryPlannerIXSelect::nodeIsSupportedBySparseIndex(const MatchExpression* queryExpr, + bool isInElemMatch) { + // The only types of queries which may not be supported by a sparse index are ones which have + // an equality to null (or an {$exists: false}), because of the language's "null or missing" + // semantics. {$exists: false} gets translated into a negation query (which a sparse index + // cannot answer), so this function only needs to check if the query performs an equality to + // null. + + // Equality to null inside an $elemMatch implies a match on literal 'null'. + if (isInElemMatch) { + return true; + } + + // Otherwise, we can't use a sparse index for $eq (or $lte, or $gte) with a null element. + // + // We can use a sparse index for $_internalExprEq with a null element. Expression language + // equality-to-null semantics are that only literal nulls match. Sparse indexes contain + // index keys for literal nulls, but not for missing elements. + const auto typ = queryExpr->matchType(); + if (typ == MatchExpression::EQ) { + const auto* queryExprEquality = static_cast<const EqualityMatchExpression*>(queryExpr); + return !queryExprEquality->getData().isNull(); + } else if (queryExpr->matchType() == MatchExpression::MATCH_IN) { + const auto* queryExprIn = static_cast<const InMatchExpression*>(queryExpr); + return !queryExprIn->hasNull(); + } + + return true; +} + +bool QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex(const MatchExpression* queryExpr) { + // AllPaths indexes only store index keys for "leaf" nodes in an object. That is, they do not + // store keys for nested objects, meaning that any kind of comparison to an object or array + // cannot be answered by the index (including with a $in). + + if (ComparisonMatchExpression::isComparisonMatchExpression(queryExpr)) { + const ComparisonMatchExpression* cmpExpr = + static_cast<const ComparisonMatchExpression*>(queryExpr); + + return canUseAllPathsIndex(cmpExpr->getData(), cmpExpr->matchType()); + } else if (queryExpr->matchType() == MatchExpression::MATCH_IN) { + const auto* queryExprIn = static_cast<const InMatchExpression*>(queryExpr); + + return std::all_of( + queryExprIn->getEqualities().begin(), + queryExprIn->getEqualities().end(), + [](const BSONElement& elt) { return canUseAllPathsIndex(elt, MatchExpression::EQ); }); + } + + return true; +} + // static // This is the public method which does not accept an ElemMatchContext. void QueryPlannerIXSelect::rateIndices(MatchExpression* node, diff --git a/src/mongo/db/query/planner_ixselect.h b/src/mongo/db/query/planner_ixselect.h index e21e88b838a..01123e652b8 100644 --- a/src/mongo/db/query/planner_ixselect.h +++ b/src/mongo/db/query/planner_ixselect.h @@ -151,6 +151,17 @@ public: static std::vector<IndexEntry> expandIndexes(const stdx::unordered_set<std::string>& fields, const std::vector<IndexEntry>& relevantIndices); + /** + * Check if this match expression is a leaf and is supported by an allPaths index. + */ + static bool nodeIsSupportedByAllPathsIndex(const MatchExpression* queryExpr); + + /* + * Return true if the given match expression can use a sparse index, false otherwise. This will + * not traverse the children of the given match expression. + */ + static bool nodeIsSupportedBySparseIndex(const MatchExpression* queryExpr, bool isInElemMatch); + private: /** * Used to keep track of if any $elemMatch predicates were encountered when walking a diff --git a/src/mongo/db/query/planner_ixselect_test.cpp b/src/mongo/db/query/planner_ixselect_test.cpp index c65700b7a60..a886bdfef6b 100644 --- a/src/mongo/db/query/planner_ixselect_test.cpp +++ b/src/mongo/db/query/planner_ixselect_test.cpp @@ -1510,4 +1510,47 @@ TEST(QueryPlannerIXSelectTest, AllPathsIndicesIncludeMatchingInternalNodes) { ASSERT_TRUE(indexEntryKeyPatternsMatch(&expectedKeyPatterns, &result)); } +TEST(QueryPlannerIXSelectTest, AllPathsIndexSupported) { + ASSERT_FALSE(QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex( + parseMatchExpression(fromjson("{x: {abc: 1}}")).get())); + ASSERT_FALSE(QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex( + parseMatchExpression(fromjson("{x: {$lt: {abc: 1}}}")).get())); + ASSERT_FALSE(QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex( + parseMatchExpression(fromjson("{x: {$in: [1, 2, 3, {abc: 1}]}}")).get())); +} + +TEST(QueryPlannerIXSelectTest, AllPathsIndexSupportedDoesNotTraverse) { + // The function will not traverse a node's children. + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex( + parseMatchExpression(fromjson("{$or: [{z: {abc: 1}}]}")).get())); + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex( + parseMatchExpression(fromjson("{x: 5, y: {abc: 1}}")).get())); + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedByAllPathsIndex( + parseMatchExpression(fromjson("{x: {$ne: {abc: 1}}}")).get())); +} + +TEST(QueryPlannerIXSelectTest, SparseIndexSupported) { + auto filterAObj = fromjson("{x: null}"); + const auto queryA = parseMatchExpression(filterAObj); + ASSERT_FALSE(QueryPlannerIXSelect::nodeIsSupportedBySparseIndex(queryA.get(), false)); + // When in an elemMatch, a comparison to null implies literal null, so it is always supported. + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedBySparseIndex(queryA.get(), true)); + + auto filterBObj = fromjson("{x: {$in: [1, 2, 3, null]}}"); + const auto queryB = parseMatchExpression(filterBObj); + ASSERT_FALSE(QueryPlannerIXSelect::nodeIsSupportedBySparseIndex(queryB.get(), false)); + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedBySparseIndex(queryB.get(), true)); +} + +TEST(QueryPlannerIXSelectTest, SparseIndexSupportedDoesNotTraverse) { + // The function will not traverse a node's children. + const bool inElemMatch = false; + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedBySparseIndex( + parseMatchExpression(fromjson("{$or: [{z: null}]}")).get(), inElemMatch)); + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedBySparseIndex( + parseMatchExpression(fromjson("{x: 5, y: null}")).get(), inElemMatch)); + ASSERT_TRUE(QueryPlannerIXSelect::nodeIsSupportedBySparseIndex( + parseMatchExpression(fromjson("{x: {$ne: null}}")).get(), inElemMatch)); +} + } // namespace diff --git a/src/mongo/db/query/query_planner_all_paths_index_test.cpp b/src/mongo/db/query/query_planner_all_paths_index_test.cpp index 3c6a3db9523..8655bdc4352 100644 --- a/src/mongo/db/query/query_planner_all_paths_index_test.cpp +++ b/src/mongo/db/query/query_planner_all_paths_index_test.cpp @@ -895,6 +895,97 @@ TEST_F(QueryPlannerTest, MultipleAllPathsIndexesHintWithPartialFilter) { assertNumSolutions(0U); } +TEST_F(QueryPlannerAllPathsTest, AllPathsIndexesDoNotSupportObjectEquality) { + addIndex(BSON("$**" << 1)); + + runQuery(fromjson("{x: {abc: 1}}")); + assertHasOnlyCollscan(); + + runQuery(fromjson("{$or: [{z: {abc: 1}}]}")); + assertHasOnlyCollscan(); + + // We can only use the index for the predicate on 'x'. + runQuery(fromjson("{x: 5, y: {abc: 1}}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {$_path: 1, x: 1}}}}}"); +} + +TEST_F(QueryPlannerAllPathsTest, AllPathsIndexesDoNotSupportObjectInequality) { + addIndex(BSON("$**" << 1)); + + runQuery(fromjson("{x: {$lt: {abc: 1}}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$lte: {abc: 1}}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$gte: {abc: 1}}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$gt: {abc: 1}}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {ne: {abc: 1}}}")); + assertHasOnlyCollscan(); + + runQuery(fromjson("{x: {$lt: [1, 2, 'a string']}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$lte: [1, 2, 'a string']}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$gte: [1, 2, 'a string']}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$gt: [1, 2, 'a string']}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {ne: [1, 2, 'a string']}}")); + assertHasOnlyCollscan(); + + runQuery(fromjson("{$or: [{z: {$ne: {abc: 1}}}]}")); + assertHasOnlyCollscan(); + + runQuery(fromjson("{$and: [{x: 5}, {$or: [{x: 1}, {y: {abc: 1}}]}]}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {$_path: 1, x: 1}}}}}"); +} + +TEST_F(QueryPlannerAllPathsTest, AllPathsIndexesDoNotSupportInWithUnsupportedValues) { + addIndex(BSON("$**" << 1)); + + runQuery(fromjson("{x: {$in: [1, 2, 3, {abc: 1}]}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$in: [1, 2, 3, ['a', 'b', 'c']]}}")); + assertHasOnlyCollscan(); + runQuery(fromjson("{x: {$in: [1, 2, 3, null]}}")); + assertHasOnlyCollscan(); +} + +TEST_F(QueryPlannerAllPathsTest, AllPathsIndexesSupportElemMatchWithNull) { + addIndex(BSON("$**" << 1)); + + // Simple case. + runQuery(fromjson("{x: {$elemMatch: {$lt: 5, $gt: 0}}}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {$_path: 1, x: 1}}}}}"); + + // null inside an $in inside an $elemMatch is supported by the allPaths index, since it means + // we're searching for an explicit null value. + runQuery(fromjson("{x: {$elemMatch: {$in: [1, 2, 3, null]}}}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {$_path: 1, x: 1}}}}}"); +} + +TEST_F(QueryPlannerAllPathsTest, AllPathsIndexesDoNotSupportElemMatchWithUnsupportedValues) { + runQuery(fromjson("{x: {$elemMatch: {$eq: ['a', 'b', 'c']}}}")); + assertHasOnlyCollscan(); + + // An object or array inside an $in inside a $elemMatch is not supported by the index. + runQuery(fromjson("{x: {$elemMatch: {$in: [1, 2, 3, {a: 1}]}}}")); + assertHasOnlyCollscan(); + + runQuery(fromjson("{x: {$elemMatch: {$in: [1, 2, 3, ['a', 'b', 'c']]}}}")); + assertHasOnlyCollscan(); +} + +TEST_F(QueryPlannerAllPathsTest, AllPathsIndexesDoNotSupportElemMatchObject) { + runQuery(fromjson("{x: {$elemMatch: {a: 1}}}")); + assertHasOnlyCollscan(); +} + // TODO SERVER-35335: Add testing for Min/Max. // TODO SERVER-36517: Add testing for DISTINCT_SCAN. // TODO SERVER-35331: Add testing for hints. diff --git a/src/mongo/db/query/query_planner_test.cpp b/src/mongo/db/query/query_planner_test.cpp index 2fc05b9f918..dc037e27a4a 100644 --- a/src/mongo/db/query/query_planner_test.cpp +++ b/src/mongo/db/query/query_planner_test.cpp @@ -2556,6 +2556,39 @@ TEST_F(QueryPlannerTest, NegationInElemMatchDoesNotUseSparseIndex) { assertHasOnlyCollscan(); } +TEST_F(QueryPlannerTest, SparseIndexCannotSupportEqualsNull) { + addIndex(BSON("i" << 1), + false, // multikey + true // sparse + ); + + runQuery(fromjson("{i: {$eq: null}}")); + assertHasOnlyCollscan(); +} + +// TODO: SERVER-37164: The semantics of {$gte: null} and {$lte: null} are inconsistent with and +// without a sparse index. It is unclear whether or not a sparse index _should_ be able to support +// these operations. +TEST_F(QueryPlannerTest, SparseIndexCanSupportGTEOrLTENull) { + params.options &= ~QueryPlannerParams::INCLUDE_COLLSCAN; + addIndex(BSON("i" << 1), + false, // multikey + true // sparse + ); + + runQuery(fromjson("{i: {$gte: null}}")); + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {i: {$gte: null}}, node: {ixscan: {pattern: " + "{i: 1}, bounds: {i: [[null,null,true,true]]}}}}}"); + + runQuery(fromjson("{i: {$lte: null}}")); + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {i: {$lte: null}}, node: {ixscan: {pattern: " + "{i: 1}, bounds: {i: [[null,null,true,true]]}}}}}"); +} + // // Regex // |