/** * Copyright (C) 2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, * as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * * As a special exception, the copyright holders give permission to link the * code of portions of this program with the OpenSSL library under certain * conditions as described in each individual source file and distribute * linked combinations including the program with the OpenSSL library. You * must comply with the GNU Affero General Public License in all respects * for all of the code used other than as permitted herein. If you modify * file(s) with this exception, you may extend this exception to your * version of the file(s), but you are not obligated to do so. If you do not * wish to do so, delete this exception statement from your version. If you * delete this exception statement from all source files in the program, * then also delete it in the license file. */ /** * This file tests db/query/plan_ranker.cpp and db/query/multi_plan_runner.cpp. */ #include "mongo/platform/basic.h" #include #include "mongo/client/dbclientcursor.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/client.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/exec/multi_plan.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/json.h" #include "mongo/db/matcher/extensions_callback_disallow_extensions.h" #include "mongo/db/namespace_string.h" #include "mongo/db/query/get_executor.h" #include "mongo/db/query/query_knobs.h" #include "mongo/db/query/query_planner.h" #include "mongo/db/query/query_planner_test_lib.h" #include "mongo/db/query/stage_builder.h" #include "mongo/dbtests/dbtests.h" #include "mongo/stdx/memory.h" namespace mongo { // How we access the external setParameter testing bool. extern AtomicBool internalQueryForceIntersectionPlans; extern AtomicBool internalQueryPlannerEnableHashIntersection; } // namespace mongo namespace PlanRankingTests { using std::unique_ptr; using std::vector; static const NamespaceString nss("unittests.PlanRankingTests"); class PlanRankingTestBase { public: PlanRankingTestBase() : _internalQueryForceIntersectionPlans(internalQueryForceIntersectionPlans.load()), _enableHashIntersection(internalQueryPlannerEnableHashIntersection.load()), _client(&_opCtx) { // Run all tests with hash-based intersection enabled. internalQueryPlannerEnableHashIntersection.store(true); // Ensure N is significantly larger then internalQueryPlanEvaluationWorks. ASSERT_GTE(N, internalQueryPlanEvaluationWorks.load() + 1000); OldClientWriteContext ctx(&_opCtx, nss.ns()); _client.dropCollection(nss.ns()); } virtual ~PlanRankingTestBase() { // Restore external setParameter testing bools. internalQueryForceIntersectionPlans.store(_internalQueryForceIntersectionPlans); internalQueryPlannerEnableHashIntersection.store(_enableHashIntersection); } void insert(const BSONObj& obj) { OldClientWriteContext ctx(&_opCtx, nss.ns()); _client.insert(nss.ns(), obj); } void addIndex(const BSONObj& obj) { ASSERT_OK(dbtests::createIndex(&_opCtx, nss.ns(), obj)); } /** * Use the MultiPlanRunner to pick the best plan for the query 'cq'. Goes through * normal planning to generate solutions and feeds them to the MPR. * * Does NOT take ownership of 'cq'. Caller DOES NOT own the returned QuerySolution*. */ QuerySolution* pickBestPlan(CanonicalQuery* cq) { AutoGetCollectionForReadCommand ctx(&_opCtx, nss); Collection* collection = ctx.getCollection(); QueryPlannerParams plannerParams; fillOutPlannerParams(&_opCtx, collection, cq, &plannerParams); // Turn this off otherwise it pops up in some plans. plannerParams.options &= ~QueryPlannerParams::KEEP_MUTATIONS; // Plan. vector solutions; Status status = QueryPlanner::plan(*cq, plannerParams, &solutions); ASSERT(status.isOK()); ASSERT_GREATER_THAN_OR_EQUALS(solutions.size(), 1U); // Fill out the MPR. _mps.reset(new MultiPlanStage(&_opCtx, collection, cq)); unique_ptr ws(new WorkingSet()); // Put each solution from the planner into the MPR. for (size_t i = 0; i < solutions.size(); ++i) { PlanStage* root; ASSERT(StageBuilder::build(&_opCtx, collection, *cq, *solutions[i], ws.get(), &root)); // Takes ownership of all (actually some) arguments. _mps->addPlan(solutions[i], root, ws.get()); } // This is what sets a backup plan, should we test for it. PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD, _opCtx.getServiceContext()->getFastClockSource()); _mps->pickBestPlan(&yieldPolicy).transitional_ignore(); ASSERT(_mps->bestPlanChosen()); size_t bestPlanIdx = _mps->bestPlanIdx(); ASSERT_LESS_THAN(bestPlanIdx, solutions.size()); // And return a pointer to the best solution. return _mps->bestSolution(); } /** * Was a backup plan picked during the ranking process? */ bool hasBackupPlan() const { ASSERT(NULL != _mps.get()); return _mps->hasBackupPlan(); } OperationContext* opCtx() { return &_opCtx; } protected: // A large number, which must be larger than the number of times // candidate plans are worked by the multi plan runner. Used for // determining the number of documents in the tests below. const int N = 12000; const ServiceContext::UniqueOperationContext _txnPtr = cc().makeOperationContext(); OperationContext& _opCtx = *_txnPtr; private: // Holds the value of global "internalQueryForceIntersectionPlans" setParameter flag. // Restored at end of test invocation regardless of test result. bool _internalQueryForceIntersectionPlans; // Holds the value of the global set parameter so it can be restored at the end // of the test. bool _enableHashIntersection; unique_ptr _mps; DBDirectClient _client; }; /** * Test that the "prefer ixisect" parameter works. */ class PlanRankingIntersectOverride : public PlanRankingTestBase { public: void run() { // 'a' is very selective, 'b' is not. for (int i = 0; i < N; ++i) { insert(BSON("a" << i << "b" << 1)); } // Add indices on 'a' and 'b'. addIndex(BSON("a" << 1)); addIndex(BSON("b" << 1)); unique_ptr cq; // Run the query {a:4, b:1}. { auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << 100 << "b" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); verify(statusWithCQ.isOK()); cq = std::move(statusWithCQ.getValue()); ASSERT(cq.get()); } // {a:100} is super selective so choose that. QuerySolution* soln = pickBestPlan(cq.get()); ASSERT(QueryPlannerTestLib::solutionMatches( "{fetch: {filter: {b:1}, node: {ixscan: {pattern: {a: 1}}}}}", soln->root.get())); // Turn on the "force intersect" option. // This will be reverted by PlanRankingTestBase's destructor when the test completes. internalQueryForceIntersectionPlans.store(true); // And run the same query again. { auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << 100 << "b" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); verify(statusWithCQ.isOK()); cq = std::move(statusWithCQ.getValue()); } // With the "ranking picks ixisect always" option we pick an intersection plan that uses // both the {a:1} and {b:1} indices even though it performs poorly. soln = pickBestPlan(cq.get()); ASSERT( QueryPlannerTestLib::solutionMatches("{fetch: {node: {andSorted: {nodes: [" "{ixscan: {filter: null, pattern: {a:1}}}," "{ixscan: {filter: null, pattern: {b:1}}}]}}}}", soln->root.get())); } }; /** * Test that a hashed AND solution plan is picked along with a non-blocking backup solution. */ class PlanRankingIntersectWithBackup : public PlanRankingTestBase { public: void run() { // 'a' is very selective, 'b' is not. for (int i = 0; i < N; ++i) { insert(BSON("a" << i << "b" << 1)); } // Add indices on 'a' and 'b'. addIndex(BSON("a" << 1)); addIndex(BSON("b" << 1)); // Run the query {a:1, b:{$gt:1}. auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << 1 << "b" << BSON("$gt" << 1))); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); verify(statusWithCQ.isOK()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); // Turn on the "force intersect" option. // This will be reverted by PlanRankingTestBase's destructor when the test completes. internalQueryForceIntersectionPlans.store(true); QuerySolution* soln = pickBestPlan(cq.get()); ASSERT( QueryPlannerTestLib::solutionMatches("{fetch: {node: {andHash: {nodes: [" "{ixscan: {filter: null, pattern: {a:1}}}," "{ixscan: {filter: null, pattern: {b:1}}}]}}}}", soln->root.get())); // Confirm that a backup plan is available. ASSERT(hasBackupPlan()); } }; /** * Two plans hit EOF at the same time, but one is covered. Make sure that we prefer the covered * plan. */ class PlanRankingPreferCovered : public PlanRankingTestBase { public: void run() { // Insert data {a:i, b:i}. Index {a:1} and {a:1, b:1}, query on 'a', projection on 'a' // and 'b'. Should prefer the second index as we can pull the 'b' data out. for (int i = 0; i < N; ++i) { insert(BSON("a" << i << "b" << i)); } addIndex(BSON("a" << 1)); addIndex(BSON("a" << 1 << "b" << 1)); // Query for a==27 with projection that wants 'a' and 'b'. auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << 27)); qr->setProj(BSON("_id" << 0 << "a" << 1 << "b" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); QuerySolution* soln = pickBestPlan(cq.get()); // Prefer the fully covered plan. ASSERT(QueryPlannerTestLib::solutionMatches( "{proj: {spec: {_id:0, a:1, b:1}, node: {ixscan: {pattern: {a: 1, b:1}}}}}", soln->root.get())); } }; /** * No plan produces any results or hits EOF. In this case we should never choose an index * intersection solution. */ class PlanRankingAvoidIntersectIfNoResults : public PlanRankingTestBase { public: void run() { // We insert lots of copies of {a:1, b:1, c: 20}. We have the indices {a:1} and {b:1}, // and the query is {a:1, b:1, c: 999}. No data that matches the query but we won't // know that during plan ranking. We don't want to choose an intersection plan here. for (int i = 0; i < N; ++i) { insert(BSON("a" << 1 << "b" << 1 << "c" << 20)); } addIndex(BSON("a" << 1)); addIndex(BSON("b" << 1)); // There is no data that matches this query but we don't know that until EOF. auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << 1 << "b" << 1 << "c" << 99)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); QuerySolution* soln = pickBestPlan(cq.get()); // Anti-prefer the intersection plan. bool bestIsScanOverA = QueryPlannerTestLib::solutionMatches( "{fetch: {node: {ixscan: {pattern: {a: 1}}}}}", soln->root.get()); bool bestIsScanOverB = QueryPlannerTestLib::solutionMatches( "{fetch: {node: {ixscan: {pattern: {b: 1}}}}}", soln->root.get()); ASSERT(bestIsScanOverA || bestIsScanOverB); } }; /** * No plan produces any results or hits EOF. In this case we should prefer covered solutions to * non-covered solutions. */ class PlanRankingPreferCoveredEvenIfNoResults : public PlanRankingTestBase { public: void run() { // We insert lots of copies of {a:1, b:1}. We have the indices {a:1} and {a:1, b:1}, // the query is for a doc that doesn't exist, but there is a projection over 'a' and // 'b'. We should prefer the index that provides a covered query. for (int i = 0; i < N; ++i) { insert(BSON("a" << 1 << "b" << 1)); } addIndex(BSON("a" << 1)); addIndex(BSON("a" << 1 << "b" << 1)); // There is no data that matches this query ({a:2}). Both scans will hit EOF before // returning any data. auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << 2)); qr->setProj(BSON("_id" << 0 << "a" << 1 << "b" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); QuerySolution* soln = pickBestPlan(cq.get()); // Prefer the fully covered plan. ASSERT(QueryPlannerTestLib::solutionMatches( "{proj: {spec: {_id:0, a:1, b:1}, node: {ixscan: {pattern: {a: 1, b:1}}}}}", soln->root.get())); } }; /** * We have an index on "a" which is somewhat selective and an index on "b" which is highly * selective (will cause an immediate EOF). Make sure that a query with predicates on both "a" * and "b" will use the index on "b". */ class PlanRankingPreferImmediateEOF : public PlanRankingTestBase { public: void run() { // 'a' is very selective, 'b' is not. for (int i = 0; i < N; ++i) { insert(BSON("a" << i << "b" << 1)); } // Add indices on 'a' and 'b'. addIndex(BSON("a" << 1)); addIndex(BSON("b" << 1)); // Run the query {a:N+1, b:1}. (No such document.) auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << N + 1 << "b" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); verify(statusWithCQ.isOK()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); // {a: 100} is super selective so choose that. QuerySolution* soln = pickBestPlan(cq.get()); ASSERT(QueryPlannerTestLib::solutionMatches( "{fetch: {filter: {b:1}, node: {ixscan: {pattern: {a: 1}}}}}", soln->root.get())); } }; /** * Same as PlanRankingPreferImmediateEOF, but substitute a range predicate on "a" for the * equality predicate on "a". The presence of the range predicate has an impact on the * intersection plan that is raced against the single-index plans: since "a" no longer generates * point interval bounds, the results of the index scan aren't guaranteed to be returned in * RecordId order, and so the intersection plan uses the AND_HASHED stage instead of the * AND_SORTED stage. It is still the case that the query should pick the plan that uses index * "b", instead of the plan that uses index "a" or the (hashed) intersection plan. */ class PlanRankingPreferImmediateEOFAgainstHashed : public PlanRankingTestBase { public: void run() { // 'a' is very selective, 'b' is not. for (int i = 0; i < N; ++i) { insert(BSON("a" << i << "b" << 1)); } // Add indices on 'a' and 'b'. addIndex(BSON("a" << 1)); addIndex(BSON("b" << 1)); // Run the query {a:N+1, b:1}. (No such document.) auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << BSON("$gte" << N + 1) << "b" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); verify(statusWithCQ.isOK()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); // {a: 100} is super selective so choose that. QuerySolution* soln = pickBestPlan(cq.get()); ASSERT(QueryPlannerTestLib::solutionMatches( "{fetch: {filter: {b:1}, node: {ixscan: {pattern: {a: 1}}}}}", soln->root.get())); } }; /** * We have an index on _id and a query over _id with a sort. Ensure that we don't pick a * collscan as the best plan even though the _id-scanning solution doesn't produce any results. */ class PlanRankingNoCollscan : public PlanRankingTestBase { public: void run() { for (int i = 0; i < N; ++i) { insert(BSON("_id" << i)); } addIndex(BSON("_id" << 1)); // Run a query with a sort. The blocking sort won't produce any data during the // evaluation period. auto qr = stdx::make_unique(nss); qr->setFilter(BSON("_id" << BSON("$gte" << 20 << "$lte" << 200))); qr->setSort(BSON("c" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); unique_ptr cq = std::move(statusWithCQ.getValue()); QuerySolution* soln = pickBestPlan(cq.get()); // The best must not be a collscan. ASSERT(QueryPlannerTestLib::solutionMatches( "{sort: {pattern: {c: 1}, limit: 0, node: {sortKeyGen: {node:" "{fetch: {filter: null, node: " "{ixscan: {filter: null, pattern: {_id: 1}}}}}}}}}", soln->root.get())); } }; /** * No indices are available, output a collscan. */ class PlanRankingCollscan : public PlanRankingTestBase { public: void run() { // Insert data for which we have no index. for (int i = 0; i < N; ++i) { insert(BSON("foo" << i)); } // Look for A Space Odyssey. auto qr = stdx::make_unique(nss); qr->setFilter(BSON("foo" << 2001)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); verify(statusWithCQ.isOK()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); QuerySolution* soln = pickBestPlan(cq.get()); // The best must be a collscan. ASSERT(QueryPlannerTestLib::solutionMatches("{cscan: {dir: 1, filter: {foo: 2001}}}", soln->root.get())); } }; /** * When no other information is available, prefer solutions without * a blocking sort stage. */ class PlanRankingAvoidBlockingSort : public PlanRankingTestBase { public: void run() { for (int i = 0; i < N; ++i) { insert(BSON("a" << 1 << "d" << i)); } // The index {d: 1, e: 1} provides the desired sort order, // while index {a: 1, b: 1} can be used to answer the // query predicate, but does not provide the sort. addIndex(BSON("a" << 1 << "b" << 1)); addIndex(BSON("d" << 1 << "e" << 1)); // Query: find({a: 1}).sort({d: 1}) auto qr = stdx::make_unique(nss); qr->setFilter(BSON("a" << 1)); qr->setSort(BSON("d" << 1)); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); // No results will be returned during the trial period, // so we expect to choose {d: 1, e: 1}, as it allows us // to avoid the sort stage. QuerySolution* soln = pickBestPlan(cq.get()); ASSERT( QueryPlannerTestLib::solutionMatches("{fetch: {filter: {a:1}, node: " "{ixscan: {filter: null, pattern: {d:1,e:1}}}}}", soln->root.get())); } }; /** * Make sure we run candidate plans for long enough when none of the * plans are producing results. */ class PlanRankingWorkPlansLongEnough : public PlanRankingTestBase { public: void run() { for (int i = 0; i < N; ++i) { insert(BSON("a" << 1)); insert(BSON("a" << 1 << "b" << 1 << "c" << i)); } // Indices on 'a' and 'b'. addIndex(BSON("a" << 1)); addIndex(BSON("b" << 1)); // Solutions using either 'a' or 'b' will take a long time to start producing // results. However, an index scan on 'b' will start producing results sooner // than an index scan on 'a'. auto qr = stdx::make_unique(nss); qr->setFilter(fromjson("{a: 1, b: 1, c: {$gte: 5000}}")); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); // Use index on 'b'. QuerySolution* soln = pickBestPlan(cq.get()); ASSERT(QueryPlannerTestLib::solutionMatches("{fetch: {node: {ixscan: {pattern: {b: 1}}}}}", soln->root.get())); } }; /** * Suppose we have two plans which are roughly equivalent, other than that * one uses an index which involves doing a lot more skipping of index keys. * Prefer the plan which does not have to do this index key skipping. */ class PlanRankingAccountForKeySkips : public PlanRankingTestBase { public: void run() { for (int i = 0; i < 100; ++i) { insert(BSON("a" << i << "b" << i << "c" << i)); } // These indices look equivalent to the ranker for the query below unless we account // for key skipping. We should pick index {a: 1} if we account for key skipping // properly. addIndex(BSON("b" << 1 << "c" << 1)); addIndex(BSON("a" << 1)); auto qr = stdx::make_unique(nss); qr->setFilter(fromjson("{a: 9, b: {$ne: 10}, c: 9}")); auto statusWithCQ = CanonicalQuery::canonicalize( opCtx(), std::move(qr), ExtensionsCallbackDisallowExtensions()); ASSERT_OK(statusWithCQ.getStatus()); unique_ptr cq = std::move(statusWithCQ.getValue()); ASSERT(NULL != cq.get()); // Expect to use index {a: 1, b: 1}. QuerySolution* soln = pickBestPlan(cq.get()); ASSERT(QueryPlannerTestLib::solutionMatches("{fetch: {node: {ixscan: {pattern: {a: 1}}}}}", soln->root.get())); } }; class All : public Suite { public: All() : Suite("query_plan_ranking") {} void setupTests() { add(); add(); add(); add(); add(); add(); add(); add(); add(); add(); add(); add(); } }; SuiteInstance planRankingAll; } // namespace PlanRankingTest