/** * Copyright (C) 2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, * as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * * As a special exception, the copyright holders give permission to link the * code of portions of this program with the OpenSSL library under certain * conditions as described in each individual source file and distribute * linked combinations including the program with the OpenSSL library. You * must comply with the GNU Affero General Public License in all respects * for all of the code used other than as permitted herein. If you modify * file(s) with this exception, you may extend this exception to your * version of the file(s), but you are not obligated to do so. If you do not * wish to do so, delete this exception statement from your version. If you * delete this exception statement from all source files in the program, * then also delete it in the license file. */ #include "mongo/platform/basic.h" #include #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/client.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/exec/subplan.h" #include "mongo/db/jsobj.h" #include "mongo/db/json.h" #include "mongo/db/matcher/extensions_callback_disallow_extensions.h" #include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/get_executor.h" #include "mongo/dbtests/dbtests.h" namespace QueryStageSubplan { static const NamespaceString nss("unittests.QueryStageSubplan"); class QueryStageSubplanBase { public: QueryStageSubplanBase() : _client(&_opCtx) {} virtual ~QueryStageSubplanBase() { OldClientWriteContext ctx(&_opCtx, nss.ns()); _client.dropCollection(nss.ns()); } void addIndex(const BSONObj& obj) { ASSERT_OK(dbtests::createIndex(&_opCtx, nss.ns(), obj)); } void insert(const BSONObj& doc) { _client.insert(nss.ns(), doc); } OperationContext* opCtx() { return &_opCtx; } protected: /** * Parses the json string 'findCmd', specifying a find command, to a CanonicalQuery. */ std::unique_ptr cqFromFindCommand(const std::string& findCmd) { BSONObj cmdObj = fromjson(findCmd); bool isExplain = false; auto qr = unittest::assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain)); auto cq = unittest::assertGet( CanonicalQuery::canonicalize(opCtx(), std::move(qr), ExtensionsCallbackNoop())); return cq; } const ServiceContext::UniqueOperationContext _txnPtr = cc().makeOperationContext(); OperationContext& _opCtx = *_txnPtr; ClockSource* _clock = _opCtx.getServiceContext()->getFastClockSource(); private: DBDirectClient _client; }; /** * SERVER-15012: test that the subplan stage does not crash when the winning solution * for an $or clause uses a '2d' index. We don't produce cache data for '2d'. The subplanner * should gracefully fail after finding that no cache data is available, allowing us to fall * back to regular planning. */ class QueryStageSubplanGeo2dOr : public QueryStageSubplanBase { public: void run() { OldClientWriteContext ctx(&_opCtx, nss.ns()); addIndex(BSON("a" << "2d" << "b" << 1)); addIndex(BSON("a" << "2d")); BSONObj query = fromjson( "{$or: [{a: {$geoWithin: {$centerSphere: [[0,0],10]}}}," "{a: {$geoWithin: {$centerSphere: [[1,1],10]}}}]}"); auto qr = stdx::make_unique(nss); qr->setFilter(query); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); std::unique_ptr cq = std::move(statusWithCQ.getValue()); Collection* collection = ctx.getCollection(); // Get planner params. QueryPlannerParams plannerParams; fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams); WorkingSet ws; std::unique_ptr subplan( new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); // Plan selection should succeed due to falling back on regular planning. PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD, _clock); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); } }; /** * Test the SubplanStage's ability to plan an individual branch using the plan cache. */ class QueryStageSubplanPlanFromCache : public QueryStageSubplanBase { public: void run() { OldClientWriteContext ctx(&_opCtx, nss.ns()); addIndex(BSON("a" << 1)); addIndex(BSON("a" << 1 << "b" << 1)); addIndex(BSON("c" << 1)); for (int i = 0; i < 10; i++) { insert(BSON("a" << 1 << "b" << i << "c" << i)); } // This query should result in a plan cache entry for the first $or branch, because // there are two competing indices. The second branch has only one relevant index, so // its winning plan should not be cached. BSONObj query = fromjson("{$or: [{a: 1, b: 3}, {c: 1}]}"); Collection* collection = ctx.getCollection(); auto qr = stdx::make_unique(nss); qr->setFilter(query); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); std::unique_ptr cq = std::move(statusWithCQ.getValue()); // Get planner params. QueryPlannerParams plannerParams; fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams); WorkingSet ws; std::unique_ptr subplan( new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD, _clock); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); // Nothing is in the cache yet, so neither branch should have been planned from // the plan cache. ASSERT_FALSE(subplan->branchPlannedFromCache(0)); ASSERT_FALSE(subplan->branchPlannedFromCache(1)); // If we repeat the same query, the plan for the first branch should have come from // the cache. ws.clear(); subplan.reset(new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); ASSERT_TRUE(subplan->branchPlannedFromCache(0)); ASSERT_FALSE(subplan->branchPlannedFromCache(1)); } }; /** * Ensure that the subplan stage doesn't create a plan cache entry if there are no query results. */ class QueryStageSubplanDontCacheZeroResults : public QueryStageSubplanBase { public: void run() { OldClientWriteContext ctx(&_opCtx, nss.ns()); addIndex(BSON("a" << 1 << "b" << 1)); addIndex(BSON("a" << 1)); addIndex(BSON("c" << 1)); for (int i = 0; i < 10; i++) { insert(BSON("a" << 1 << "b" << i << "c" << i)); } // Running this query should not create any cache entries. For the first branch, it's // because there are no matching results. For the second branch it's because there is only // one relevant index. BSONObj query = fromjson("{$or: [{a: 1, b: 15}, {c: 1}]}"); Collection* collection = ctx.getCollection(); auto qr = stdx::make_unique(nss); qr->setFilter(query); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); std::unique_ptr cq = std::move(statusWithCQ.getValue()); // Get planner params. QueryPlannerParams plannerParams; fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams); WorkingSet ws; std::unique_ptr subplan( new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD, _clock); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); // Nothing is in the cache yet, so neither branch should have been planned from // the plan cache. ASSERT_FALSE(subplan->branchPlannedFromCache(0)); ASSERT_FALSE(subplan->branchPlannedFromCache(1)); // If we run the query again, it should again be the case that neither branch gets planned // from the cache (because the first call to pickBestPlan() refrained from creating any // cache entries). ws.clear(); subplan.reset(new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); ASSERT_FALSE(subplan->branchPlannedFromCache(0)); ASSERT_FALSE(subplan->branchPlannedFromCache(1)); } }; /** * Ensure that the subplan stage doesn't create a plan cache entry if there are no query results. */ class QueryStageSubplanDontCacheTies : public QueryStageSubplanBase { public: void run() { OldClientWriteContext ctx(&_opCtx, nss.ns()); addIndex(BSON("a" << 1 << "b" << 1)); addIndex(BSON("a" << 1 << "c" << 1)); addIndex(BSON("d" << 1)); for (int i = 0; i < 10; i++) { insert(BSON("a" << 1 << "e" << 1 << "d" << 1)); } // Running this query should not create any cache entries. For the first branch, it's // because plans using the {a: 1, b: 1} and {a: 1, c: 1} indices should tie during plan // ranking. For the second branch it's because there is only one relevant index. BSONObj query = fromjson("{$or: [{a: 1, e: 1}, {d: 1}]}"); Collection* collection = ctx.getCollection(); auto qr = stdx::make_unique(nss); qr->setFilter(query); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); std::unique_ptr cq = std::move(statusWithCQ.getValue()); // Get planner params. QueryPlannerParams plannerParams; fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams); WorkingSet ws; std::unique_ptr subplan( new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD, _clock); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); // Nothing is in the cache yet, so neither branch should have been planned from // the plan cache. ASSERT_FALSE(subplan->branchPlannedFromCache(0)); ASSERT_FALSE(subplan->branchPlannedFromCache(1)); // If we run the query again, it should again be the case that neither branch gets planned // from the cache (because the first call to pickBestPlan() refrained from creating any // cache entries). ws.clear(); subplan.reset(new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); ASSERT_FALSE(subplan->branchPlannedFromCache(0)); ASSERT_FALSE(subplan->branchPlannedFromCache(1)); } }; /** * Unit test the subplan stage's canUseSubplanning() method. */ class QueryStageSubplanCanUseSubplanning : public QueryStageSubplanBase { public: void run() { // We won't try and subplan something that doesn't have an $or. { std::string findCmd = "{find: 'testns', filter: {$and:[{a:1}, {b:1}]}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Don't try and subplan if there is no filter. { std::string findCmd = "{find: 'testns'}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // We won't try and subplan two contained ORs. { std::string findCmd = "{find: 'testns'," "filter: {$or:[{a:1}, {b:1}], $or:[{c:1}, {d:1}], e:1}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning if there is a hint. { std::string findCmd = "{find: 'testns'," "filter: {$or: [{a:1, b:1}, {c:1, d:1}]}," "hint: {a:1, b:1}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning with min. { std::string findCmd = "{find: 'testns'," "filter: {$or: [{a:1, b:1}, {c:1, d:1}]}," "min: {a:1, b:1}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning with max. { std::string findCmd = "{find: 'testns'," "filter: {$or: [{a:1, b:1}, {c:1, d:1}]}," "max: {a:2, b:2}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning with tailable. { std::string findCmd = "{find: 'testns'," "filter: {$or: [{a:1, b:1}, {c:1, d:1}]}," "tailable: true}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning with snapshot. { std::string findCmd = "{find: 'testns'," "filter: {$or: [{a:1, b:1}, {c:1, d:1}]}," "snapshot: true}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can use subplanning for rooted $or. { std::string findCmd = "{find: 'testns'," "filter: {$or: [{a:1, b:1}, {c:1, d:1}]}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_TRUE(SubplanStage::canUseSubplanning(*cq)); std::string findCmd2 = "{find: 'testns'," "filter: {$or: [{a:1}, {c:1}]}}"; std::unique_ptr cq2 = cqFromFindCommand(findCmd2); ASSERT_TRUE(SubplanStage::canUseSubplanning(*cq2)); } // Can't use subplanning for a single contained $or. // // TODO: Consider allowing this to use subplanning (see SERVER-13732). { std::string findCmd = "{find: 'testns'," "filter: {e: 1, $or: [{a:1, b:1}, {c:1, d:1}]}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning if the contained $or query has a geo predicate. // // TODO: Consider allowing this to use subplanning (see SERVER-13732). { std::string findCmd = "{find: 'testns'," "filter: {loc: {$geoWithin: {$centerSphere: [[0,0], 1]}}," "e: 1, $or: [{a:1, b:1}, {c:1, d:1}]}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning if the contained $or query also has a $text predicate. { std::string findCmd = "{find: 'testns'," "filter: {$text: {$search: 'foo'}," "e: 1, $or: [{a:1, b:1}, {c:1, d:1}]}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } // Can't use subplanning if the contained $or query also has a $near predicate. { std::string findCmd = "{find: 'testns'," "filter: {loc: {$near: [0, 0]}," "e: 1, $or: [{a:1, b:1}, {c:1, d:1}]}}"; std::unique_ptr cq = cqFromFindCommand(findCmd); ASSERT_FALSE(SubplanStage::canUseSubplanning(*cq)); } } }; /** * Unit test the subplan stage's rewriteToRootedOr() method. */ class QueryStageSubplanRewriteToRootedOr : public QueryStageSubplanBase { public: void run() { // Rewrite (AND (OR a b) e) => (OR (AND a e) (AND b e)) { BSONObj queryObj = fromjson("{$or:[{a:1}, {b:1}], e:1}"); const CollatorInterface* collator = nullptr; StatusWithMatchExpression expr = MatchExpressionParser::parse( queryObj, ExtensionsCallbackDisallowExtensions(), collator); ASSERT_OK(expr.getStatus()); std::unique_ptr rewrittenExpr = SubplanStage::rewriteToRootedOr(std::move(expr.getValue())); std::string findCmdRewritten = "{find: 'testns'," "filter: {$or:[{a:1,e:1}, {b:1,e:1}]}}"; std::unique_ptr cqRewritten = cqFromFindCommand(findCmdRewritten); ASSERT(rewrittenExpr->equivalent(cqRewritten->root())); } // Rewrite (AND (OR a b) e f) => (OR (AND a e f) (AND b e f)) { BSONObj queryObj = fromjson("{$or:[{a:1}, {b:1}], e:1, f:1}"); const CollatorInterface* collator = nullptr; StatusWithMatchExpression expr = MatchExpressionParser::parse( queryObj, ExtensionsCallbackDisallowExtensions(), collator); ASSERT_OK(expr.getStatus()); std::unique_ptr rewrittenExpr = SubplanStage::rewriteToRootedOr(std::move(expr.getValue())); std::string findCmdRewritten = "{find: 'testns'," "filter: {$or:[{a:1,e:1,f:1}, {b:1,e:1,f:1}]}}"; std::unique_ptr cqRewritten = cqFromFindCommand(findCmdRewritten); ASSERT(rewrittenExpr->equivalent(cqRewritten->root())); } // Rewrite (AND (OR (AND a b) (AND c d) e f) => (OR (AND a b e f) (AND c d e f)) { BSONObj queryObj = fromjson("{$or:[{a:1,b:1}, {c:1,d:1}], e:1,f:1}"); const CollatorInterface* collator = nullptr; StatusWithMatchExpression expr = MatchExpressionParser::parse( queryObj, ExtensionsCallbackDisallowExtensions(), collator); ASSERT_OK(expr.getStatus()); std::unique_ptr rewrittenExpr = SubplanStage::rewriteToRootedOr(std::move(expr.getValue())); std::string findCmdRewritten = "{find: 'testns'," "filter: {$or:[{a:1,b:1,e:1,f:1}," "{c:1,d:1,e:1,f:1}]}}"; std::unique_ptr cqRewritten = cqFromFindCommand(findCmdRewritten); ASSERT(rewrittenExpr->equivalent(cqRewritten->root())); } } }; /** * Test the subplan stage's ability to answer a contained $or query. */ class QueryStageSubplanPlanContainedOr : public QueryStageSubplanBase { public: void run() { OldClientWriteContext ctx(&_opCtx, nss.ns()); addIndex(BSON("b" << 1 << "a" << 1)); addIndex(BSON("c" << 1 << "a" << 1)); BSONObj query = fromjson("{a: 1, $or: [{b: 2}, {c: 3}]}"); // Two of these documents match. insert(BSON("_id" << 1 << "a" << 1 << "b" << 2)); insert(BSON("_id" << 2 << "a" << 2 << "b" << 2)); insert(BSON("_id" << 3 << "a" << 1 << "c" << 3)); insert(BSON("_id" << 4 << "a" << 1 << "c" << 4)); auto qr = stdx::make_unique(nss); qr->setFilter(query); auto cq = unittest::assertGet(CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions())); Collection* collection = ctx.getCollection(); // Get planner params. QueryPlannerParams plannerParams; fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams); WorkingSet ws; std::unique_ptr subplan( new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); // Plan selection should succeed due to falling back on regular planning. PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD, _clock); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); // Work the stage until it produces all results. size_t numResults = 0; PlanStage::StageState stageState = PlanStage::NEED_TIME; while (stageState != PlanStage::IS_EOF) { WorkingSetID id = WorkingSet::INVALID_ID; stageState = subplan->work(&id); ASSERT_NE(stageState, PlanStage::DEAD); ASSERT_NE(stageState, PlanStage::FAILURE); if (stageState == PlanStage::ADVANCED) { ++numResults; WorkingSetMember* member = ws.get(id); ASSERT(member->hasObj()); ASSERT(SimpleBSONObjComparator::kInstance.evaluate( member->obj.value() == BSON("_id" << 1 << "a" << 1 << "b" << 2)) || SimpleBSONObjComparator::kInstance.evaluate( member->obj.value() == BSON("_id" << 3 << "a" << 1 << "c" << 3))); } } ASSERT_EQ(numResults, 2U); } }; /** * Test the subplan stage's ability to answer a rooted $or query with a $ne and a sort. * * Regression test for SERVER-19388. */ class QueryStageSubplanPlanRootedOrNE : public QueryStageSubplanBase { public: void run() { OldClientWriteContext ctx(&_opCtx, nss.ns()); addIndex(BSON("a" << 1 << "b" << 1)); addIndex(BSON("a" << 1 << "c" << 1)); // Every doc matches. insert(BSON("_id" << 1 << "a" << 1)); insert(BSON("_id" << 2 << "a" << 2)); insert(BSON("_id" << 3 << "a" << 3)); insert(BSON("_id" << 4)); auto qr = stdx::make_unique(nss); qr->setFilter(fromjson("{$or: [{a: 1}, {a: {$ne:1}}]}")); qr->setSort(BSON("d" << 1)); auto cq = unittest::assertGet(CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions())); Collection* collection = ctx.getCollection(); QueryPlannerParams plannerParams; fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams); WorkingSet ws; std::unique_ptr subplan( new SubplanStage(&_opCtx, collection, &ws, plannerParams, cq.get())); PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD, _clock); ASSERT_OK(subplan->pickBestPlan(&yieldPolicy)); size_t numResults = 0; PlanStage::StageState stageState = PlanStage::NEED_TIME; while (stageState != PlanStage::IS_EOF) { WorkingSetID id = WorkingSet::INVALID_ID; stageState = subplan->work(&id); ASSERT_NE(stageState, PlanStage::DEAD); ASSERT_NE(stageState, PlanStage::FAILURE); if (stageState == PlanStage::ADVANCED) { ++numResults; } } ASSERT_EQ(numResults, 4U); } }; class All : public Suite { public: All() : Suite("query_stage_subplan") {} void setupTests() { add(); add(); add(); add(); add(); add(); add(); add(); } }; SuiteInstance all; } // namespace QueryStageSubplan