diff options
author | Craig Harris <craig.harris@10gen.com> | 2014-05-13 16:51:01 -0400 |
---|---|---|
committer | Craig Harris <craig.harris@10gen.com> | 2014-05-13 18:01:22 -0400 |
commit | 6ada135a2dfb937106736e37885efc08dadc23f9 (patch) | |
tree | 200e003eb70328eb2ca5e7375ab603b71d7ccdaf /src/mongo | |
parent | 5368b375128302c1bb32ec7c5e6d5b7e6c5cd38b (diff) | |
download | mongo-6ada135a2dfb937106736e37885efc08dadc23f9.tar.gz |
SERVER-13674: Refactor CachedPlanRunner and MultiPlanRunner as stages
Diffstat (limited to 'src/mongo')
21 files changed, 988 insertions, 1168 deletions
diff --git a/src/mongo/db/exec/SConscript b/src/mongo/db/exec/SConscript index d1e3015ae85..037fe773e7c 100644 --- a/src/mongo/db/exec/SConscript +++ b/src/mongo/db/exec/SConscript @@ -40,6 +40,7 @@ env.Library( "2dnear.cpp", "and_hash.cpp", "and_sorted.cpp", + "cached_plan.cpp", "collection_scan.cpp", "count.cpp", "distinct_scan.cpp", @@ -48,6 +49,7 @@ env.Library( "keep_mutations.cpp", "limit.cpp", "merge_sort.cpp", + "multi_plan.cpp", "oplogstart.cpp", "or.cpp", "projection.cpp", diff --git a/src/mongo/db/exec/cached_plan.cpp b/src/mongo/db/exec/cached_plan.cpp new file mode 100644 index 00000000000..a18b2132890 --- /dev/null +++ b/src/mongo/db/exec/cached_plan.cpp @@ -0,0 +1,162 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/cached_plan.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/util/mongoutils/str.h" + +// for updateCache +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/qlog.h" + +namespace mongo { + + CachedPlanStage::CachedPlanStage(const Collection* collection, + CanonicalQuery* cq, + PlanStage* mainChild, + PlanStage* backupChild) + : _collection(collection), + _canonicalQuery(cq), + _mainChildPlan(mainChild), + _backupChildPlan(backupChild), + _usingBackupChild(false), + _alreadyProduced(false), + _updatedCache(false) { } + + CachedPlanStage::~CachedPlanStage() { + // We may have produced all necessary results without hitting EOF. + // In this case, we still want to update the cache with feedback. + if (!_updatedCache) { + updateCache(); + } + } + + bool CachedPlanStage::isEOF() { return getActiveChild()->isEOF(); } + + PlanStage::StageState CachedPlanStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (isEOF()) { return PlanStage::IS_EOF; } + + StageState childStatus = getActiveChild()->work(out); + + if (PlanStage::ADVANCED == childStatus) { + // we'll skip backupPlan processing now + _alreadyProduced = true; + } + else if (PlanStage::IS_EOF == childStatus) { + updateCache(); + } + else if (PlanStage::FAILURE == childStatus + && !_alreadyProduced + && !_usingBackupChild + && NULL != _backupChildPlan.get()) { + _usingBackupChild = true; + childStatus = _backupChildPlan->work(out); + } + return childStatus; + } + + void CachedPlanStage::prepareToYield() { + if (! _usingBackupChild) { + _mainChildPlan->prepareToYield(); + } + + if (NULL != _backupChildPlan.get()) { + _backupChildPlan->prepareToYield(); + } + ++_commonStats.yields; + } + + void CachedPlanStage::recoverFromYield() { + + if (NULL != _backupChildPlan.get()) { + _backupChildPlan->recoverFromYield(); + } + + if (! _usingBackupChild) { + _mainChildPlan->recoverFromYield(); + } + ++_commonStats.unyields; + } + + void CachedPlanStage::invalidate(const DiskLoc& dl, InvalidationType type) { + if (! _usingBackupChild) { + _mainChildPlan->invalidate(dl, type); + } + if (NULL != _backupChildPlan.get()) { + _backupChildPlan->invalidate(dl, type); + } + ++_commonStats.invalidates; + } + + PlanStageStats* CachedPlanStage::getStats() { + _commonStats.isEOF = isEOF(); + + auto_ptr<PlanStageStats> ret(new PlanStageStats(_commonStats, STAGE_CACHED_PLAN)); + ret->specific.reset(new CachedPlanStats(_specificStats)); + + if (_usingBackupChild) { + ret->children.push_back(_backupChildPlan->getStats()); + } + else { + ret->children.push_back(_mainChildPlan->getStats()); + } + + return ret.release(); + } + + void CachedPlanStage::updateCache() { + _updatedCache = true; + + std::auto_ptr<PlanCacheEntryFeedback> feedback(new PlanCacheEntryFeedback()); + feedback->stats.reset(getStats()); + feedback->score = PlanRanker::scoreTree(feedback->stats.get()); + + PlanCache* cache = _collection->infoCache()->getPlanCache(); + Status fbs = cache->feedback(*_canonicalQuery, feedback.release()); + + if (!fbs.isOK()) { + QLOG() << _canonicalQuery->ns() << ": Failed to update cache with feedback: " + << fbs.toString() << " - " + << "(query: " << _canonicalQuery->getQueryObj() + << "; sort: " << _canonicalQuery->getParsed().getSort() + << "; projection: " << _canonicalQuery->getParsed().getProj() + << ") is no longer in plan cache."; + } + } + + PlanStage* CachedPlanStage::getActiveChild() const { + return _usingBackupChild ? _backupChildPlan.get() : _mainChildPlan.get(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/cached_plan.h b/src/mongo/db/exec/cached_plan.h new file mode 100644 index 00000000000..7bdd16f4f6a --- /dev/null +++ b/src/mongo/db/exec/cached_plan.h @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/canonical_query.h" + +namespace mongo { + + /** + * This stage outputs its mainChild, and possibly its backup child + * and also updates the cache. + * + * Preconditions: Valid DiskLoc. + * + */ + class CachedPlanStage : public PlanStage { + public: + CachedPlanStage(const Collection* collection, + CanonicalQuery* cq, + PlanStage* mainChild, + PlanStage* backupChild=NULL); + + virtual ~CachedPlanStage(); + + virtual bool isEOF(); + + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + PlanStage* getActiveChild() const; + void updateCache(); + + // not owned + const Collection* _collection; + + // not owned + CanonicalQuery* _canonicalQuery; + + // owned by us + boost::scoped_ptr<PlanStage> _mainChildPlan; + boost::scoped_ptr<PlanStage> _backupChildPlan; + + // True if the main plan errors before producing results + // and if a backup plan is available (can happen with blocking sorts) + bool _usingBackupChild; + + // True if the childPlan has produced results yet. + bool _alreadyProduced; + + // Have we updated the cache with our plan stats yet? + bool _updatedCache; + + // Stats + CommonStats _commonStats; + CachedPlanStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/multi_plan.cpp b/src/mongo/db/exec/multi_plan.cpp new file mode 100644 index 00000000000..5e558e444e5 --- /dev/null +++ b/src/mongo/db/exec/multi_plan.cpp @@ -0,0 +1,446 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/multi_plan.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/util/mongoutils/str.h" + +// for updateCache +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/qlog.h" + +namespace mongo { + MultiPlanStage::MultiPlanStage(const Collection* collection, CanonicalQuery* cq) + : _collection(collection), + _query(cq), + _bestPlanIdx(kNoSuchPlan), + _backupPlanIdx(kNoSuchPlan), + _failure(false), + _failureCount(0), + _statusMemberId(WorkingSet::INVALID_ID) { } + + MultiPlanStage::~MultiPlanStage() { + if (bestPlanChosen()) { + delete _candidates[_bestPlanIdx].root; + + // for now, the runner that executes this multi-plan-stage wants to own + // the query solution for the best plan. So we won't delete it here. + // eventually, plan stages may own their query solutions. + // + // delete _candidates[_bestPlanIdx].solution; // (owned by containing runner) + + if (hasBackupPlan()) { + delete _candidates[_backupPlanIdx].solution; + delete _candidates[_backupPlanIdx].root; + } + } + else { + for (size_t ix = 0; ix < _candidates.size(); ++ix) { + delete _candidates[ix].solution; + delete _candidates[ix].root; + } + } + + for (vector<PlanStageStats*>::iterator it = _candidateStats.begin(); + it != _candidateStats.end(); + ++it) { + delete *it; + } + } + + void MultiPlanStage::addPlan(QuerySolution* solution, PlanStage* root, + WorkingSet* ws) { + _candidates.push_back(CandidatePlan(solution, root, ws)); + } + + bool MultiPlanStage::isEOF() { + if (_failure) { return true; } + + // If _bestPlanIdx hasn't been found, can't be at EOF + if (!bestPlanChosen()) { return false; } + + // We must have returned all our cached results + // and there must be no more results from the best plan. + CandidatePlan& bestPlan = _candidates[_bestPlanIdx]; + return bestPlan.results.empty() && bestPlan.root->isEOF(); + } + + PlanStage::StageState MultiPlanStage::work(WorkingSetID* out) { + if (_failure) { + *out = _statusMemberId; + return PlanStage::FAILURE; + } + + CandidatePlan& bestPlan = _candidates[_bestPlanIdx]; + + // Look for an already produced result that provides the data the caller wants. + if (!bestPlan.results.empty()) { + *out = bestPlan.results.front(); + bestPlan.results.pop_front(); + return PlanStage::ADVANCED; + } + + // best plan had no (or has no more) cached results + + StageState state = bestPlan.root->work(out); + + if (PlanStage::FAILURE == state && hasBackupPlan()) { + QLOG() << "Best plan errored out switching to backup\n"; + // Uncache the bad solution if we fall back + // on the backup solution. + // + // XXX: Instead of uncaching we should find a way for the + // cached plan runner to fall back on a different solution + // if the best solution fails. Alternatively we could try to + // defer cache insertion to be after the first produced result. + + _collection->infoCache()->getPlanCache()->remove(*_query); + + _bestPlanIdx = _backupPlanIdx; + _backupPlanIdx = kNoSuchPlan; + + return _candidates[_bestPlanIdx].root->work(out); + } + + if (hasBackupPlan() && PlanStage::ADVANCED == state) { + QLOG() << "Best plan had a blocking sort, became unblocked, deleting backup plan\n"; + delete _candidates[_backupPlanIdx].solution; + delete _candidates[_backupPlanIdx].root; + _backupPlanIdx = kNoSuchPlan; + } + + return state; + } + + void MultiPlanStage::pickBestPlan() { + // Run each plan some number of times. This number is at least as great as + // 'internalQueryPlanEvaluationWorks', but may be larger for big collections. + size_t numWorks = internalQueryPlanEvaluationWorks; + if (NULL != _collection) { + // For large collections, the number of works is set to be this + // fraction of the collection size. + double fraction = internalQueryPlanEvaluationCollFraction; + + numWorks = std::max(size_t(internalQueryPlanEvaluationWorks), + size_t(fraction * _collection->numRecords())); + } + + // Work the plans, stopping when a plan hits EOF or returns some + // fixed number of results. + for (size_t ix = 0; ix < numWorks; ++ix) { + bool moreToDo = workAllPlans(); + if (!moreToDo) { break; } + } + + if (_failure) { return; } + + // After picking best plan, ranking will own plan stats from + // candidate solutions (winner and losers). + std::auto_ptr<PlanRankingDecision> ranking(new PlanRankingDecision); + _bestPlanIdx = PlanRanker::pickBestPlan(_candidates, ranking.get()); + verify(_bestPlanIdx >= 0 && _bestPlanIdx < static_cast<int>(_candidates.size())); + + // Copy candidate order. We will need this to sort candidate stats for explain + // after transferring ownership of 'ranking' to plan cache. + std::vector<size_t> candidateOrder = ranking->candidateOrder; + + CandidatePlan& bestCandidate = _candidates[_bestPlanIdx]; + std::list<WorkingSetID>& alreadyProduced = bestCandidate.results; + QuerySolution* bestSolution = bestCandidate.solution; + + QLOG() << "Winning solution:\n" << bestSolution->toString() << endl; + LOG(2) << "Winning plan: " << getPlanSummary(*bestSolution); + + _backupPlanIdx = kNoSuchPlan; + if (bestSolution->hasBlockingStage && (0 == alreadyProduced.size())) { + QLOG() << "Winner has blocking stage, looking for backup plan...\n"; + for (size_t ix = 0; ix < _candidates.size(); ++ix) { + if (!_candidates[ix].solution->hasBlockingStage) { + QLOG() << "Candidate " << ix << " is backup child\n"; + _backupPlanIdx = ix; + break; + } + } + } + + // Store the choice we just made in the cache. We do + // not cache the query if: + // 1) The query is of a type that is not safe to cache, or + // 2) the winning plan did not actually produce any results, + // without hitting EOF. In this case, we have no information to + // suggest that this plan is good. + const PlanStageStats* bestStats = ranking->stats.vector()[0]; + if (PlanCache::shouldCacheQuery(*_query) + && (!alreadyProduced.empty() || bestStats->common.isEOF)) { + + // Create list of candidate solutions for the cache with + // the best solution at the front. + std::vector<QuerySolution*> solutions; + + // Generate solutions and ranking decisions sorted by score. + for (size_t orderingIndex = 0; + orderingIndex < candidateOrder.size(); ++orderingIndex) { + // index into candidates/ranking + size_t ix = candidateOrder[orderingIndex]; + solutions.push_back(_candidates[ix].solution); + } + + // Check solution cache data. Do not add to cache if + // we have any invalid SolutionCacheData data. + // XXX: One known example is 2D queries + bool validSolutions = true; + for (size_t ix = 0; ix < solutions.size(); ++ix) { + if (NULL == solutions[ix]->cacheData.get()) { + QLOG() << "Not caching query because this solution has no cache data: " + << solutions[ix]->toString(); + validSolutions = false; + break; + } + } + + if (validSolutions) { + _collection->infoCache()->getPlanCache()->add(*_query, solutions, ranking.release()); + } + } + + // Clear out the candidate plans, leaving only stats as we're all done w/them. + // Traverse candidate plans in order or score + for (size_t orderingIndex = 0; + orderingIndex < candidateOrder.size(); ++orderingIndex) { + // index into candidates/ranking + int ix = candidateOrder[orderingIndex]; + + if (ix == _bestPlanIdx) { continue; } + if (ix == _backupPlanIdx) { continue; } + + delete _candidates[ix].solution; + + // Remember the stats for the candidate plan because we always show it on an + // explain. (The {verbose:false} in explain() is client-side trick; we always + // generate a "verbose" explain.) + PlanStageStats* stats = _candidates[ix].root->getStats(); + if (stats) { + _candidateStats.push_back(stats); + } + delete _candidates[ix].root; + } + } + + bool MultiPlanStage::workAllPlans() { + bool doneWorking = false; + + for (size_t ix = 0; ix < _candidates.size(); ++ix) { + CandidatePlan& candidate = _candidates[ix]; + if (candidate.failed) { continue; } + + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState state = candidate.root->work(&id); + + if (PlanStage::ADVANCED == state) { + // Save result for later. + candidate.results.push_back(id); + + // Once a plan returns enough results, stop working. + if (candidate.results.size() + >= size_t(internalQueryPlanEvaluationMaxResults)) { + doneWorking = true; + } + } + else if (PlanStage::NEED_FETCH == state) { + // id has a loc and refers to an obj we need to fetch. + WorkingSetMember* member = candidate.ws->get(id); + + // This must be true for somebody to request a fetch and can only change when an + // invalidation happens, which is when we give up a lock. Don't give up the + // lock between receiving the NEED_FETCH and actually fetching(?). + verify(member->hasLoc()); + + // XXX: remove NEED_FETCH + } + else if (PlanStage::IS_EOF == state) { + // First plan to hit EOF wins automatically. Stop evaluating other plans. + // Assumes that the ranking will pick this plan. + doneWorking = true; + } + else if (PlanStage::NEED_TIME != state) { + // FAILURE or DEAD. Do we want to just tank that plan and try the rest? We + // probably want to fail globally as this shouldn't happen anyway. + + candidate.failed = true; + ++_failureCount; + + // Propagate most recent seen failure to parent. + if (PlanStage::FAILURE == state) { + BSONObj objOut; + WorkingSetCommon::getStatusMemberObject(*candidate.ws, id, &objOut); + _statusMemberId = id; + } + + if (_failureCount == _candidates.size()) { + _failure = true; + return false; + } + } + } + + return !doneWorking; + } + + void MultiPlanStage::prepareToYield() { + if (_failure) return; + + // this logic is from multi_plan_runner + // but does it really make sense to operate on + // the _bestPlan if we've switched to the backup? + + if (bestPlanChosen()) { + _candidates[_bestPlanIdx].root->prepareToYield(); + if (hasBackupPlan()) { + _candidates[_backupPlanIdx].root->prepareToYield(); + } + } + else { + allPlansSaveState(); + } + } + + void MultiPlanStage::recoverFromYield() { + if (_failure) return; + + // this logic is from multi_plan_runner + // but does it really make sense to operate on + // the _bestPlan if we've switched to the backup? + + if (bestPlanChosen()) { + _candidates[_bestPlanIdx].root->recoverFromYield(); + if (hasBackupPlan()) { + _candidates[_backupPlanIdx].root->recoverFromYield(); + } + } + else { + allPlansRestoreState(); + } + } + + namespace { + void invalidateHelper( + WorkingSet* ws, // may flag for review + const DiskLoc& dl, + list<WorkingSetID>* idsToInvalidate, + const Collection* collection + ) { + for (list<WorkingSetID>::iterator it = idsToInvalidate->begin(); + it != idsToInvalidate->end();) { + WorkingSetMember* member = ws->get(*it); + if (member->hasLoc() && member->loc == dl) { + list<WorkingSetID>::iterator next = it; + next++; + WorkingSetCommon::fetchAndInvalidateLoc(member, collection); + ws->flagForReview(*it); + idsToInvalidate->erase(it); + it = next; + } + else { + it++; + } + } + } + } + + void MultiPlanStage::invalidate(const DiskLoc& dl, InvalidationType type) { + if (_failure) { return; } + + if (bestPlanChosen()) { + CandidatePlan& bestPlan = _candidates[_bestPlanIdx]; + bestPlan.root->invalidate(dl, type); + invalidateHelper(bestPlan.ws, dl, &bestPlan.results, _collection); + if (hasBackupPlan()) { + CandidatePlan& backupPlan = _candidates[_backupPlanIdx]; + backupPlan.root->invalidate(dl, type); + invalidateHelper(backupPlan.ws, dl, &backupPlan.results, _collection); + } + } + else { + for (size_t ix = 0; ix < _candidates.size(); ++ix) { + _candidates[ix].root->invalidate(dl, type); + invalidateHelper(_candidates[ix].ws, dl, &_candidates[ix].results, _collection); + } + } + } + + bool MultiPlanStage::hasBackupPlan() const { + return kNoSuchPlan != _backupPlanIdx; + } + + bool MultiPlanStage::bestPlanChosen() const { + return kNoSuchPlan != _bestPlanIdx; + } + + int MultiPlanStage::bestPlanIdx() const { + return _bestPlanIdx; + } + + QuerySolution* MultiPlanStage::bestSolution() { + if (_bestPlanIdx == kNoSuchPlan) + return NULL; + + return _candidates[_bestPlanIdx].solution; + } + + void MultiPlanStage::allPlansSaveState() { + for (size_t i = 0; i < _candidates.size(); ++i) { + _candidates[i].root->prepareToYield(); + } + } + + void MultiPlanStage::allPlansRestoreState() { + for (size_t i = 0; i < _candidates.size(); ++i) { + _candidates[i].root->recoverFromYield(); + } + } + + PlanStageStats* MultiPlanStage::getStats() { + if (bestPlanChosen()) { + return _candidates[_bestPlanIdx].root->getStats(); + } + if (hasBackupPlan()) { + return _candidates[_backupPlanIdx].root->getStats(); + } + _commonStats.isEOF = isEOF(); + + auto_ptr<PlanStageStats> ret(new PlanStageStats(_commonStats, STAGE_MULTI_PLAN)); + + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/multi_plan_runner.h b/src/mongo/db/exec/multi_plan.h index 5da33d74653..8281857518e 100644 --- a/src/mongo/db/query/multi_plan_runner.h +++ b/src/mongo/db/exec/multi_plan.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -28,61 +28,60 @@ #pragma once -#include <boost/scoped_ptr.hpp> -#include <list> -#include <vector> - -#include "mongo/base/status.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/exec/plan_stage.h" #include "mongo/db/exec/working_set.h" -#include "mongo/db/query/plan_ranker.h" // for CandidatePlan -#include "mongo/db/query/runner.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/db/query/plan_ranker.h" namespace mongo { - class BSONObj; - class CanonicalQuery; - class DiskLoc; - class PlanExecutor; - class PlanStage; - struct QuerySolution; - class TypeExplain; - struct PlanInfo; - class WorkingSet; - /** - * Runs several plans in parallel and picks the best one. Caches the selection for future use. + * This stage outputs its mainChild, and possibly it's backup child + * and also updates the cache. + * + * Preconditions: Valid DiskLoc. + * */ - class MultiPlanRunner : public Runner { + class MultiPlanStage : public PlanStage { public: - /** - * Takes ownership of query. - */ - MultiPlanRunner(const Collection* collection, CanonicalQuery* query); - virtual ~MultiPlanRunner(); - - /** - * Takes ownership of all arguments - */ - void addPlan(QuerySolution* solution, PlanStage* root, WorkingSet* ws); + /** Takes no ownership */ + MultiPlanStage(const Collection* collection, CanonicalQuery* cq); - /** - * Get the next result. Yielding is handled internally. If a best plan is not picked when - * this is called, we call pickBestPlan() internally. - */ - Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + virtual ~MultiPlanStage(); virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + + virtual void recoverFromYield(); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + /** Takes ownership of QuerySolution and PlanStage. not of WorkingSet */ + void addPlan(QuerySolution* solution, PlanStage* root, WorkingSet* sharedWs); + /** * Runs all plans added by addPlan, ranks them, and picks a best. Deletes all loser plans. * All further calls to getNext(...) will return results from the best plan. - * - * Returns true if a best plan was picked, false if there was an error. - * If there was a failure in the underlying plan, *objOut may hold error details. - * - * If out is not-NULL, set *out to the index of the picked plan. */ - bool pickBestPlan(size_t* out, BSONObj* objOut); + void pickBestPlan(); + + /** Return true if a best plan has been chosen */ + bool bestPlanChosen() const; + + /** Return the index of the best plan chosen, for testing */ + int bestPlanIdx() const; + + /** Returns the QuerySolution for the best plan, or NULL if no best plan */ + QuerySolution* bestSolution(); /** * Returns true if a backup plan was picked. @@ -91,45 +90,38 @@ namespace mongo { */ bool hasBackupPlan() const; - /** - * Caching the best plan is (currently implemented as) a destructive act so we separate it - * from ranking so that inspection of the winning solution is possible. Also sets a backup - * plan if a backup plan is needed. Exposed for testing. - */ - void cacheBestPlan(); - - virtual void saveState(); - virtual bool restoreState(); - virtual void invalidate(const DiskLoc& dl, InvalidationType type); - - virtual const std::string& ns(); - - virtual void kill(); - - virtual const Collection* collection() { return _collection; } - - /** - * Returns OK, allocating and filling in '*explain' and '*planInfo' with details of - * the "winner" plan. Caller takes ownership of '*explain' and '*planInfo'. Otherwise, - * return a status describing the error. - */ - virtual Status getInfo(TypeExplain** explain, - PlanInfo** planInfo) const; - private: /** * Have all our candidate plans do something. * If all our candidate plans fail, *objOut will contain * information on the failure. */ - bool workAllPlans(BSONObj* objOut); + bool workAllPlans(); void allPlansSaveState(); void allPlansRestoreState(); + static const int kNoSuchPlan = -1; + + // not owned here const Collection* _collection; - // Were we killed by an invalidate? - bool _killed; + // The query that we're trying to figure out the best solution to. + // not owned here + CanonicalQuery* _query; + + // Candidate plans. Owned here. + std::vector<CandidatePlan> _candidates; + + // Candidate plans' stats. Owned here. + std::vector<PlanStageStats*> _candidateStats; + + // index into _candidates, of the winner of the plan competition + // uses -1 / kNoSuchPlan when best plan is not (yet) known + int _bestPlanIdx; + + // index into _candidates, of the backup plan for sort + // uses -1 / kNoSuchPlan when best plan is not (yet) known + int _backupPlanIdx; // Did all plans fail while we were running them? Note that one plan can fail // during normal execution of the plan competition. Here is an example: @@ -143,37 +135,13 @@ namespace mongo { // If everything fails during the plan competition, we can't pick one. size_t _failureCount; - // The winner of the plan competition... - boost::scoped_ptr<PlanExecutor> _bestPlan; - - // ...and any results it produced while working toward winning. - std::list<WorkingSetID> _alreadyProduced; - - // ...and the solution, for caching. - boost::scoped_ptr<QuerySolution> _bestSolution; - - // Candidate plans. - std::vector<CandidatePlan> _candidates; - - // Candidate plans' stats. Owned here. - std::vector<PlanStageStats*> _candidateStats; - - // The query that we're trying to figure out the best solution to. - boost::scoped_ptr<CanonicalQuery> _query; - - // What's the ranking? Produced by pickBestPlan, consumed by cacheBestPlan. - auto_ptr<PlanRankingDecision> _ranking; - - // What's the best child? Filled out by pickBestPlan, consumed by cacheBestPlan. - size_t _bestChild; - - // - // Backup plan for sort - // + // if pickBestPlan fails, this is set to the wsid of the statusMember + // returned by ::work() + WorkingSetID _statusMemberId; - QuerySolution* _backupSolution; - PlanExecutor* _backupPlan; - std::list<WorkingSetID> _backupAlreadyProduced; + // Stats + CommonStats _commonStats; + MultiPlanStats _specificStats; }; } // namespace mongo diff --git a/src/mongo/db/exec/plan_stats.h b/src/mongo/db/exec/plan_stats.h index 536b4d822ff..a3f91c8ebd4 100644 --- a/src/mongo/db/exec/plan_stats.h +++ b/src/mongo/db/exec/plan_stats.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -183,6 +183,14 @@ namespace mongo { size_t matchTested; }; + struct CachedPlanStats : public SpecificStats { + CachedPlanStats() { } + + virtual SpecificStats* clone() const { + return new CachedPlanStats(*this); + } + }; + struct CollectionScanStats : public SpecificStats { CollectionScanStats() : docsTested(0) { } @@ -288,6 +296,14 @@ namespace mongo { }; + struct MultiPlanStats : public SpecificStats { + MultiPlanStats() { } + + virtual SpecificStats* clone() const { + return new MultiPlanStats(*this); + } + }; + struct OrStats : public SpecificStats { OrStats() : dupsTested(0), dupsDropped(0), diff --git a/src/mongo/db/exec/working_set.cpp b/src/mongo/db/exec/working_set.cpp index ec9eaca2453..3d268cfde28 100644 --- a/src/mongo/db/exec/working_set.cpp +++ b/src/mongo/db/exec/working_set.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, diff --git a/src/mongo/db/exec/working_set.h b/src/mongo/db/exec/working_set.h index b71e4f01931..90ef451f75c 100644 --- a/src/mongo/db/exec/working_set.h +++ b/src/mongo/db/exec/working_set.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index c65f0c46b6d..bf7eb904379 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -31,13 +31,11 @@ env.Library( env.Library( target='query', source=[ - "cached_plan_runner.cpp", "eof_runner.cpp", "explain_plan.cpp", "get_runner.cpp", "idhack_runner.cpp", "internal_runner.cpp", - "multi_plan_runner.cpp", "new_find.cpp", "plan_executor.cpp", "plan_ranker.cpp", diff --git a/src/mongo/db/query/cached_plan_runner.cpp b/src/mongo/db/query/cached_plan_runner.cpp deleted file mode 100644 index f93a9332ce3..00000000000 --- a/src/mongo/db/query/cached_plan_runner.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Copyright (C) 2013 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/query/cached_plan_runner.h" - -#include "mongo/db/catalog/collection.h" -#include "mongo/db/catalog/database.h" -#include "mongo/db/client.h" -#include "mongo/db/diskloc.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/exec/plan_stage.h" -#include "mongo/db/exec/working_set.h" -#include "mongo/db/query/canonical_query.h" -#include "mongo/db/query/explain_plan.h" -#include "mongo/db/query/plan_cache.h" -#include "mongo/db/query/plan_executor.h" -#include "mongo/db/query/plan_ranker.h" -#include "mongo/db/query/qlog.h" -#include "mongo/db/query/query_solution.h" -#include "mongo/db/query/type_explain.h" - -namespace mongo { - - CachedPlanRunner::CachedPlanRunner(const Collection* collection, - CanonicalQuery* canonicalQuery, - QuerySolution* solution, - PlanStage* root, - WorkingSet* ws) - : _collection(collection), - _canonicalQuery(canonicalQuery), - _solution(solution), - _exec(new PlanExecutor(ws, root, collection)), - _alreadyProduced(false), - _updatedCache(false), - _killed(false) { } - - CachedPlanRunner::~CachedPlanRunner() { - // The runner may produce all necessary results without hitting EOF. In this case, we still - // want to update the cache with feedback. - if (!_updatedCache) { - updateCache(); - } - } - - Runner::RunnerState CachedPlanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { - Runner::RunnerState state = _exec->getNext(objOut, dlOut); - - if (Runner::RUNNER_ADVANCED == state) { - // Indicate that the plan executor already produced results. - _alreadyProduced = true; - } - - // If the plan executor errors before producing any results, - // and we have a backup plan available, then fall back on the - // backup plan. This can happen if '_exec' has a blocking sort. - if (Runner::RUNNER_ERROR == state && !_alreadyProduced && NULL != _backupPlan.get()) { - _exec.reset(_backupPlan.release()); - state = _exec->getNext(objOut, dlOut); - } - - // This could be called several times and we don't want to update the cache every time. - if (Runner::RUNNER_EOF == state && !_updatedCache) { - updateCache(); - } - - return state; - } - - bool CachedPlanRunner::isEOF() { - return _exec->isEOF(); - } - - void CachedPlanRunner::saveState() { - _exec->saveState(); - if (NULL != _backupPlan.get()) { - _backupPlan->saveState(); - } - } - - bool CachedPlanRunner::restoreState() { - if (NULL != _backupPlan.get()) { - _backupPlan->restoreState(); - } - return _exec->restoreState(); - } - - void CachedPlanRunner::invalidate(const DiskLoc& dl, InvalidationType type) { - _exec->invalidate(dl, type); - if (NULL != _backupPlan.get()) { - _backupPlan->invalidate(dl, type); - } - } - - const std::string& CachedPlanRunner::ns() { - return _canonicalQuery->getParsed().ns(); - } - - void CachedPlanRunner::kill() { - _killed = true; - _collection = NULL; - _exec->kill(); - if (NULL != _backupPlan.get()) { - _backupPlan->kill(); - } - } - - Status CachedPlanRunner::getInfo(TypeExplain** explain, - PlanInfo** planInfo) const { - if (NULL != explain) { - if (NULL == _exec.get()) { - return Status(ErrorCodes::InternalError, "No plan available to provide stats"); - } - - // - // Explain for the winner plan - // - - scoped_ptr<PlanStageStats> stats(_exec->getStats()); - if (NULL == stats.get()) { - return Status(ErrorCodes::InternalError, "no stats available to explain plan"); - } - - // Alternate plans not needed for explainMultiPlain. - // We don't bother showing the alternative plans because their bounds - // may not match the bounds of the query that we're running. If a user - // is explaining a query that could have been executed in >1 way it - // won't use a cached plan runner for that reason - // getInfo is used to generate explain info for system.profile and - // slow query logging only. - // User visible explain output is generated by multi plan runner's getInfo(). - std::vector<PlanStageStats*> emptyStats; - return explainMultiPlan(*stats, emptyStats, _solution.get(), explain); - } - else if (NULL != planInfo) { - if (NULL == _solution.get()) { - return Status(ErrorCodes::InternalError, - "no best solution available for plan info"); - } - getPlanInfo(*_solution, planInfo); - } - - return Status::OK(); - } - - void CachedPlanRunner::updateCache() { - _updatedCache = true; - - if (_killed) { - return; - } - - PlanCache* cache = _collection->infoCache()->getPlanCache(); - - std::auto_ptr<PlanCacheEntryFeedback> feedback(new PlanCacheEntryFeedback()); - feedback->stats.reset(_exec->getStats()); - feedback->score = PlanRanker::scoreTree(feedback->stats.get()); - - Status fbs = cache->feedback(*_canonicalQuery, feedback.release()); - - if (!fbs.isOK()) { - QLOG() << _canonicalQuery->ns() << ": Failed to update cache with feedback: " - << fbs.toString() << " - " - << "(query: " << _canonicalQuery->getQueryObj() - << "; sort: " << _canonicalQuery->getParsed().getSort() - << "; projection: " << _canonicalQuery->getParsed().getProj() - << ") is no longer in plan cache."; - } - } - - void CachedPlanRunner::setBackupPlan(QuerySolution* qs, PlanStage* root, WorkingSet* ws) { - _backupSolution.reset(qs); - _backupPlan.reset(new PlanExecutor(ws, root, _collection)); - } - -} // namespace mongo diff --git a/src/mongo/db/query/cached_plan_runner.h b/src/mongo/db/query/cached_plan_runner.h deleted file mode 100644 index f994640dcf7..00000000000 --- a/src/mongo/db/query/cached_plan_runner.h +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright (C) 2013 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <boost/scoped_ptr.hpp> -#include <string> - -#include "mongo/base/status.h" -#include "mongo/db/query/plan_cache.h" -#include "mongo/db/query/runner.h" - -namespace mongo { - - class BSONObj; - class CachedSolution; - class CanonicalQuery; - class DiskLoc; - class PlanExecutor; - class PlanStage; - class TypeExplain; - struct PlanInfo; - class WorkingSet; - - /** - * CachedPlanRunner runs a plan retrieved from the cache. - * - * If we run a plan from the cache and behavior wildly deviates from expected behavior, we may - * remove the plan from the cache. See plan_cache.h. - */ - class CachedPlanRunner : public Runner { - public: - /** - * Takes ownership of all arguments. - */ - CachedPlanRunner(const Collection* collection, - CanonicalQuery* canonicalQuery, - QuerySolution* solution, - PlanStage* root, - WorkingSet* ws); - - virtual ~CachedPlanRunner(); - - Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); - - virtual bool isEOF(); - - virtual void saveState(); - - virtual bool restoreState(); - - virtual void invalidate(const DiskLoc& dl, InvalidationType type); - - virtual const std::string& ns(); - - virtual void kill(); - - virtual const Collection* collection() { return _collection; } - /** - * Returns OK, allocating and filling in '*explain' and '*planInfo' with details of - * the cached plan. Caller takes ownership of '*explain' and '*planInfo'. Otherwise, - * return a status describing the error. - */ - virtual Status getInfo(TypeExplain** explain, - PlanInfo** planInfo) const; - - /** - * Takes ownership of all arguments. - */ - void setBackupPlan(QuerySolution* qs, PlanStage* root, WorkingSet* ws); - - private: - void updateCache(); - - const Collection* _collection; - - boost::scoped_ptr<CanonicalQuery> _canonicalQuery; - boost::scoped_ptr<QuerySolution> _solution; - boost::scoped_ptr<PlanExecutor> _exec; - - // Owned here. If non-NULL, then this plan executor is capable - // of executing a backup plan in the case of a blocking sort. - std::auto_ptr<PlanExecutor> _backupPlan; - - // Owned here. If non-NULL, contains the query solution corresponding - // to the backup plan. - boost::scoped_ptr<QuerySolution> _backupSolution; - - // Whether the executor for the winning plan has produced results yet. - bool _alreadyProduced; - - // Have we updated the cache with our plan stats yet? - bool _updatedCache; - - // Has the runner been killed? - bool _killed; - }; - -} // namespace mongo diff --git a/src/mongo/db/query/explain_plan.cpp b/src/mongo/db/query/explain_plan.cpp index b3f27580b1f..d93c02123df 100644 --- a/src/mongo/db/query/explain_plan.cpp +++ b/src/mongo/db/query/explain_plan.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -412,6 +412,8 @@ namespace mongo { return "AND_HASH"; case STAGE_AND_SORTED: return "AND_SORTED"; + case STAGE_CACHED_PLAN: + return "CACHED_PLAN"; case STAGE_COLLSCAN: return "COLLSCAN"; case STAGE_COUNT: @@ -432,6 +434,8 @@ namespace mongo { return "KEEP_MUTATIONS"; case STAGE_LIMIT: return "LIMIT"; + case STAGE_MULTI_PLAN: + return "MULTI_PLAN"; case STAGE_OR: return "OR"; case STAGE_PROJECTION: diff --git a/src/mongo/db/query/get_runner.cpp b/src/mongo/db/query/get_runner.cpp index 97c8a2e0fa0..1c3614584a7 100644 --- a/src/mongo/db/query/get_runner.cpp +++ b/src/mongo/db/query/get_runner.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -32,7 +32,8 @@ #include "mongo/base/parse_number.h" #include "mongo/client/dbclientinterface.h" -#include "mongo/db/query/cached_plan_runner.h" +#include "mongo/db/exec/cached_plan.h" +#include "mongo/db/exec/multi_plan.h" #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/eof_runner.h" #include "mongo/db/query/explain_plan.h" @@ -40,7 +41,6 @@ #include "mongo/db/query/idhack_runner.h" #include "mongo/db/query/index_bounds_builder.h" #include "mongo/db/query/internal_plans.h" -#include "mongo/db/query/multi_plan_runner.h" #include "mongo/db/query/plan_cache.h" #include "mongo/db/query/planner_analysis.h" #include "mongo/db/query/planner_access.h" @@ -188,81 +188,8 @@ namespace mongo { plannerParams->options |= QueryPlannerParams::SPLIT_LIMITED_SORT; } - Status getRunnerFromCache(CanonicalQuery* canonicalQuery, - Collection* collection, - const QueryPlannerParams& plannerParams, - Runner** out) { - // Skip cache look up for non-cacheable queries. - if (!PlanCache::shouldCacheQuery(*canonicalQuery)) { - return Status(ErrorCodes::BadValue, "query is not cacheable"); - } - - CachedSolution* rawCS; - Status cacheLookupStatus = collection->infoCache()->getPlanCache()->get(*canonicalQuery, - &rawCS); - if (!cacheLookupStatus.isOK()) { - return cacheLookupStatus; - } - - // We have a CachedSolution. Have the planner turn it into a QuerySolution. - boost::scoped_ptr<CachedSolution> cs(rawCS); - QuerySolution *qs, *backupQs; - Status status = QueryPlanner::planFromCache(*canonicalQuery, - plannerParams, - *cs, - &qs, - &backupQs); - if (!status.isOK()) { - return status; - } - - // If our cached solution is a hit for a count query, try to turn it into a fast count - // thing. - if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) { - if (turnIxscanIntoCount(qs)) { - LOG(2) << "Using fast count: " << canonicalQuery->toStringShort() - << ", planSummary: " << getPlanSummary(*qs); - - WorkingSet* ws; - PlanStage* root; - verify(StageBuilder::build(collection, *qs, &root, &ws)); - *out = new SingleSolutionRunner(collection, - canonicalQuery, qs, root, ws); - if (NULL != backupQs) { - delete backupQs; - } - return Status::OK(); - } - } - - // If we're here, we're going to used the cached plan and things are normal. - LOG(2) << "Using cached query plan: " << canonicalQuery->toStringShort() - << ", planSummary: " << getPlanSummary(*qs); - - WorkingSet* ws; - PlanStage* root; - verify(StageBuilder::build(collection, *qs, &root, &ws)); - CachedPlanRunner* cpr = new CachedPlanRunner(collection, - canonicalQuery, - qs, - root, - ws); - - // If there's a backup solution, let the CachedPlanRunner know about it. - if (NULL != backupQs) { - WorkingSet* backupWs; - PlanStage* backupRoot; - verify(StageBuilder::build(collection, *backupQs, &backupRoot, &backupWs)); - cpr->setBackupPlan(backupQs, backupRoot, backupWs); - } - - *out = cpr; - return Status::OK(); - } - /** - * For a given query, get a runner. The runner could be a SingleSolutionRunner, a - * CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc. + * For a given query, get a runner. */ Status getRunner(Collection* collection, CanonicalQuery* rawCanonicalQuery, @@ -313,17 +240,52 @@ namespace mongo { plannerParams.options = plannerOptions; fillOutPlannerParams(collection, rawCanonicalQuery, &plannerParams); - // See if the cache has what we're looking for. - Status cacheStatus = getRunnerFromCache(canonicalQuery.get(), - collection, - plannerParams, - out); - - // This can be not-OK and we can carry on. It just means the query wasn't cached. - if (cacheStatus.isOK()) { - // We got a cached runner. - canonicalQuery.release(); - return cacheStatus; + // Try to look up a cached solution for the query. + + CachedSolution* rawCS; + if (PlanCache::shouldCacheQuery(*canonicalQuery) && + collection->infoCache()->getPlanCache()->get(*canonicalQuery, &rawCS).isOK()) { + // We have a CachedSolution. Have the planner turn it into a QuerySolution. + boost::scoped_ptr<CachedSolution> cs(rawCS); + QuerySolution *qs, *backupQs; + QuerySolution*& chosenSolution=qs; // either qs or backupQs + Status status = QueryPlanner::planFromCache(*canonicalQuery, plannerParams, *cs, + &qs, &backupQs); + + if (status.isOK()) { + // the working set will be shared by the root and backupRoot plans + // and owned by the containing single-solution-runner + // + WorkingSet* sharedWs = new WorkingSet(); + + PlanStage* root, *backupRoot=NULL; + verify(StageBuilder::build(collection, *qs, sharedWs, &root)); + if ((plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) + && turnIxscanIntoCount(qs)) { + // If our cached solution is a hit for a count query, + // try to turn it into a fast count thing. + LOG(2) << "Using fast count: " << canonicalQuery->toStringShort() + << ", planSummary: " << getPlanSummary(*qs); + + if (NULL != backupQs) { + // should all queries that can be answered by an index scan + // go through this path? e.g. if there's a compound index on A,B + // and we're looking for B where A=some-constant? + // + // if not, what's special about count? + delete backupQs; + } + verify(StageBuilder::build(collection, *backupQs, sharedWs, &backupRoot)); + } + + // add a CachedPlanStage on top of the previous root + root = new CachedPlanStage(collection, rawCanonicalQuery, root, backupRoot); + + *out = new SingleSolutionRunner(collection, + canonicalQuery.release(), + chosenSolution, root, sharedWs); + return Status::OK(); + } } if (internalQueryPlanOrChildrenIndependently @@ -388,9 +350,9 @@ namespace mongo { << ", planSummary: " << getPlanSummary(*solutions[i]); // We're not going to cache anything that's fast count. - WorkingSet* ws; + WorkingSet* ws = new WorkingSet(); PlanStage* root; - verify(StageBuilder::build(collection, *solutions[i], &root, &ws)); + verify(StageBuilder::build(collection, *solutions[i], ws, &root)); *out = new SingleSolutionRunner(collection, canonicalQuery.release(), solutions[i], @@ -407,9 +369,9 @@ namespace mongo { << ", planSummary: " << getPlanSummary(*solutions[0]); // Only one possible plan. Run it. Build the stages from the solution. - WorkingSet* ws; + WorkingSet* ws = new WorkingSet(); PlanStage* root; - verify(StageBuilder::build(collection, *solutions[0], &root, &ws)); + verify(StageBuilder::build(collection, *solutions[0], ws, &root)); // And, run the plan. *out = new SingleSolutionRunner(collection, @@ -420,20 +382,33 @@ namespace mongo { return Status::OK(); } else { - // Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on. - auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(collection,canonicalQuery.release())); + // Many solutions. Create a MultiPlanStage to pick the best, update the cache, and so on. - for (size_t i = 0; i < solutions.size(); ++i) { - WorkingSet* ws; - PlanStage* root; - if (solutions[i]->cacheData.get()) { - solutions[i]->cacheData->indexFilterApplied = plannerParams.indexFiltersApplied; + // The working set will be shared by all candidate plans and owned by the containing runner + WorkingSet* sharedWorkingSet = new WorkingSet(); + + MultiPlanStage* multiPlanStage = new MultiPlanStage(collection, rawCanonicalQuery); + + for (size_t ix = 0; ix < solutions.size(); ++ix) { + if (solutions[ix]->cacheData.get()) { + solutions[ix]->cacheData->indexFilterApplied = plannerParams.indexFiltersApplied; } - verify(StageBuilder::build(collection, *solutions[i], &root, &ws)); - // Takes ownership of all arguments. - mpr->addPlan(solutions[i], root, ws); + + // version of StageBuild::build when WorkingSet is shared + PlanStage* nextPlanRoot; + verify(StageBuilder::build(collection, *solutions[ix], + sharedWorkingSet, &nextPlanRoot)); + + // Owns none of the arguments + multiPlanStage->addPlan(solutions[ix], nextPlanRoot, sharedWorkingSet); } - *out = mpr.release(); + multiPlanStage->pickBestPlan(); + *out = new SingleSolutionRunner(collection, + canonicalQuery.release(), + multiPlanStage->bestSolution(), + multiPlanStage, + sharedWorkingSet); + return Status::OK(); } } @@ -790,9 +765,9 @@ namespace mongo { LOG(2) << "Using fast distinct: " << cq->toStringShort() << ", planSummary: " << getPlanSummary(*soln); - WorkingSet* ws; + WorkingSet* ws = new WorkingSet(); PlanStage* root; - verify(StageBuilder::build(collection, *soln, &root, &ws)); + verify(StageBuilder::build(collection, *soln, ws, &root)); *out = new SingleSolutionRunner(collection, cq, soln, root, ws); return Status::OK(); } @@ -818,9 +793,9 @@ namespace mongo { << ", planSummary: " << getPlanSummary(*solutions[i]); // Build and return the SSR over solutions[i]. - WorkingSet* ws; + WorkingSet* ws = new WorkingSet(); PlanStage* root; - verify(StageBuilder::build(collection, *solutions[i], &root, &ws)); + verify(StageBuilder::build(collection, *solutions[i], ws, &root)); *out = new SingleSolutionRunner(collection, cq, solutions[i], root, ws); return Status::OK(); } diff --git a/src/mongo/db/query/multi_plan_runner.cpp b/src/mongo/db/query/multi_plan_runner.cpp deleted file mode 100644 index 99d3a144a99..00000000000 --- a/src/mongo/db/query/multi_plan_runner.cpp +++ /dev/null @@ -1,541 +0,0 @@ -/** - * Copyright (C) 2013 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/query/multi_plan_runner.h" - -#include <algorithm> -#include <memory> - -#include "mongo/db/client.h" -#include "mongo/db/catalog/collection.h" -#include "mongo/db/catalog/database.h" -#include "mongo/db/diskloc.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/exec/plan_stage.h" -#include "mongo/db/exec/working_set_common.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/query/canonical_query.h" -#include "mongo/db/query/explain_plan.h" -#include "mongo/db/query/plan_cache.h" -#include "mongo/db/query/plan_executor.h" -#include "mongo/db/query/qlog.h" -#include "mongo/db/query/query_knobs.h" -#include "mongo/db/query/query_solution.h" -#include "mongo/db/query/type_explain.h" -#include "mongo/db/catalog/collection.h" - -namespace mongo { - - MultiPlanRunner::MultiPlanRunner(const Collection* collection, CanonicalQuery* query) - : _collection(collection), - _killed(false), - _failure(false), - _failureCount(0), - _query(query), - _bestChild(numeric_limits<size_t>::max()), - _backupSolution(NULL), - _backupPlan(NULL) { } - - MultiPlanRunner::~MultiPlanRunner() { - for (size_t i = 0; i < _candidates.size(); ++i) { - delete _candidates[i].solution; - delete _candidates[i].root; - // ws must die after the root. - delete _candidates[i].ws; - } - - if (NULL != _backupSolution) { - delete _backupSolution; - } - - if (NULL != _backupPlan) { - delete _backupPlan; - } - - for (vector<PlanStageStats*>::iterator it = _candidateStats.begin(); - it != _candidateStats.end(); - ++it) { - delete *it; - } - } - - void MultiPlanRunner::addPlan(QuerySolution* solution, PlanStage* root, WorkingSet* ws) { - _candidates.push_back(CandidatePlan(solution, root, ws)); - } - - void MultiPlanRunner::saveState() { - if (_failure || _killed) { return; } - - if (NULL != _bestPlan) { - _bestPlan->saveState(); - if (NULL != _backupPlan) { - _backupPlan->saveState(); - } - } - else { - allPlansSaveState(); - } - } - - bool MultiPlanRunner::restoreState() { - if (_failure || _killed) { return false; } - - if (NULL != _bestPlan) { - bool best = _bestPlan->restoreState(); - // backup plan is OK by default. Only can be set to not-OK if it exists and fails. - bool backup = true; - if (NULL != _backupPlan) { - backup = _backupPlan->restoreState(); - } - // We're OK to continue if the best plan and the backup plan are OK. - return best && backup; - } - else { - allPlansRestoreState(); - return true; - } - } - - void MultiPlanRunner::invalidate(const DiskLoc& dl, InvalidationType type) { - if (_failure || _killed) { return; } - - if (NULL != _bestPlan) { - _bestPlan->invalidate(dl, type); - for (list<WorkingSetID>::iterator it = _alreadyProduced.begin(); - it != _alreadyProduced.end();) { - WorkingSetMember* member = _bestPlan->getWorkingSet()->get(*it); - if (member->hasLoc() && member->loc == dl) { - list<WorkingSetID>::iterator next = it; - next++; - WorkingSetCommon::fetchAndInvalidateLoc(member, _collection); - _bestPlan->getWorkingSet()->flagForReview(*it); - _alreadyProduced.erase(it); - it = next; - } - else { - it++; - } - } - if (NULL != _backupPlan) { - _backupPlan->invalidate(dl, type); - for (list<WorkingSetID>::iterator it = _backupAlreadyProduced.begin(); - it != _backupAlreadyProduced.end();) { - WorkingSetMember* member = _backupPlan->getWorkingSet()->get(*it); - if (member->hasLoc() && member->loc == dl) { - list<WorkingSetID>::iterator next = it; - next++; - WorkingSetCommon::fetchAndInvalidateLoc(member, _collection); - _backupPlan->getWorkingSet()->flagForReview(*it); - _backupAlreadyProduced.erase(it); - it = next; - } - else { - it++; - } - } - } - } - else { - for (size_t i = 0; i < _candidates.size(); ++i) { - _candidates[i].root->invalidate(dl, type); - for (list<WorkingSetID>::iterator it = _candidates[i].results.begin(); - it != _candidates[i].results.end();) { - WorkingSetMember* member = _candidates[i].ws->get(*it); - if (member->hasLoc() && member->loc == dl) { - list<WorkingSetID>::iterator next = it; - next++; - WorkingSetCommon::fetchAndInvalidateLoc(member, _collection); - _candidates[i].ws->flagForReview(*it); - _candidates[i].results.erase(it); - it = next; - } - else { - it++; - } - } - } - } - } - - bool MultiPlanRunner::isEOF() { - if (_failure || _killed) { return true; } - // If _bestPlan is not NULL, you haven't picked the best plan yet, so you're not EOF. - if (NULL == _bestPlan) { return false; } - // We must return all our cached results and there must be no results from the best plan. - return _alreadyProduced.empty() && _bestPlan->isEOF(); - } - - const std::string& MultiPlanRunner::ns() { - return _query->getParsed().ns(); - } - - void MultiPlanRunner::kill() { - _killed = true; - _collection = NULL; - if (NULL != _bestPlan) { _bestPlan->kill(); } - if (NULL != _backupPlan) { _backupPlan->kill(); } - } - - Runner::RunnerState MultiPlanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { - if (_killed) { return Runner::RUNNER_DEAD; } - if (_failure) { return Runner::RUNNER_ERROR; } - - // If we haven't picked the best plan yet... - if (NULL == _bestPlan) { - if (!pickBestPlan(NULL, objOut)) { - verify(_failure || _killed); - if (_killed) { return Runner::RUNNER_DEAD; } - if (_failure) { return Runner::RUNNER_ERROR; } - } - cacheBestPlan(); - } - - // Look for an already produced result that provides the data the caller wants. - while (!_alreadyProduced.empty()) { - WorkingSetID id = _alreadyProduced.front(); - _alreadyProduced.pop_front(); - - WorkingSetMember* member = _bestPlan->getWorkingSet()->get(id); - - // Note that this copies code from PlanExecutor. - if (NULL != objOut) { - if (WorkingSetMember::LOC_AND_IDX == member->state) { - if (1 != member->keyData.size()) { - _bestPlan->getWorkingSet()->free(id); - // If the caller needs the key data and the WSM doesn't have it, drop the - // result and carry on. - continue; - } - *objOut = member->keyData[0].keyData; - } - else if (member->hasObj()) { - *objOut = member->obj; - } - else { - // If the caller needs an object and the WSM doesn't have it, drop and - // try the next result. - _bestPlan->getWorkingSet()->free(id); - continue; - } - } - - if (NULL != dlOut) { - if (member->hasLoc()) { - *dlOut = member->loc; - } - else { - // If the caller needs a DiskLoc and the WSM doesn't have it, drop and carry on. - _bestPlan->getWorkingSet()->free(id); - continue; - } - } - - // If we're here, the caller has all the data needed and we've set the out - // parameters. Remove the result from the WorkingSet. - _bestPlan->getWorkingSet()->free(id); - return Runner::RUNNER_ADVANCED; - } - - RunnerState state = _bestPlan->getNext(objOut, dlOut); - - if (Runner::RUNNER_ERROR == state && (NULL != _backupSolution)) { - QLOG() << "Best plan errored out; switching to backup.\n"; - // Uncache the bad solution if we fall back - // on the backup solution. - // - // XXX: Instead of uncaching we should find a way for the - // cached plan runner to fall back on a different solution - // if the best solution fails. Alternatively we could try to - // defer cache insertion to be after the first produced result. - PlanCache* cache = _collection->infoCache()->getPlanCache(); - cache->remove(*_query); - - // Move the backup info into the bestPlan info and clear the backup - // info. - _bestPlan.reset(_backupPlan); - _backupPlan = NULL; - _bestSolution.reset(_backupSolution); - _backupSolution = NULL; - _alreadyProduced = _backupAlreadyProduced; - _backupAlreadyProduced.clear(); - return getNext(objOut, dlOut); - } - - if (NULL != _backupSolution && Runner::RUNNER_ADVANCED == state) { - QLOG() << "Best plan had a blocking sort, became unblocked; deleting backup plan.\n"; - delete _backupSolution; - delete _backupPlan; - _backupSolution = NULL; - _backupPlan = NULL; - // TODO: free from WS? - _backupAlreadyProduced.clear(); - } - - return state; - } - - bool MultiPlanRunner::pickBestPlan(size_t* out, BSONObj* objOut) { - // Run each plan some number of times. This number is at least as great as - // 'internalQueryPlanEvaluationWorks', but may be larger for big collections. - size_t numWorks = internalQueryPlanEvaluationWorks; - if (NULL != _collection) { - // For large collections, the number of works is set to be this - // fraction of the collection size. - double fraction = internalQueryPlanEvaluationCollFraction; - - numWorks = std::max(size_t(internalQueryPlanEvaluationWorks), - size_t(fraction * _collection->numRecords())); - } - - // Work the plans, stopping when a plan hits EOF or returns some - // fixed number of results. - for (size_t i = 0; i < numWorks; ++i) { - bool moreToDo = workAllPlans(objOut); - if (!moreToDo) { break; } - } - - if (_failure || _killed) { return false; } - - // After picking best plan, ranking will own plan stats from - // candidate solutions (winner and losers). - _ranking.reset(new PlanRankingDecision()); - _bestChild = PlanRanker::pickBestPlan(_candidates, _ranking.get()); - if (NULL != out) { *out = _bestChild; } - return true; - } - - void MultiPlanRunner::cacheBestPlan() { - // Must call pickBestPlan before. - verify(_bestChild != numeric_limits<size_t>::max()); - - // Copy candidate order. We will need this to sort candidate stats for explain - // after transferring ownership of 'ranking' to plan cache. - std::vector<size_t> candidateOrder = _ranking->candidateOrder; - - // Run the best plan. Store it. - _bestPlan.reset(new PlanExecutor(_candidates[_bestChild].ws, - _candidates[_bestChild].root, - _collection)); - _alreadyProduced = _candidates[_bestChild].results; - _bestSolution.reset(_candidates[_bestChild].solution); - - QLOG() << "Winning solution:\n" << _bestSolution->toString() << endl; - LOG(2) << "Winning plan: " << getPlanSummary(*_bestSolution); - - size_t backupChild = _bestChild; - if (_bestSolution->hasBlockingStage && (0 == _alreadyProduced.size())) { - QLOG() << "Winner has blocking stage, looking for backup plan.\n"; - for (size_t i = 0; i < _candidates.size(); ++i) { - if (!_candidates[i].solution->hasBlockingStage) { - QLOG() << "Candidate " << i << " is backup child.\n"; - backupChild = i; - _backupSolution = _candidates[i].solution; - _backupAlreadyProduced = _candidates[i].results; - _backupPlan = new PlanExecutor(_candidates[i].ws, - _candidates[i].root, - _collection); - break; - } - } - } - - // Store the choice we just made in the cache. We do - // not cache the query if: - // 1) The query is of a type that is not safe to cache, or - // 2) the winning plan did not actually produce any results, - // without hitting EOF. In this case, we have no information to - // suggest that this plan is good. - const PlanStageStats* bestStats = _ranking->stats.vector()[0]; - if (PlanCache::shouldCacheQuery(*_query) - && (!_alreadyProduced.empty() || bestStats->common.isEOF)) { - PlanCache* cache = _collection->infoCache()->getPlanCache(); - // Create list of candidate solutions for the cache with - // the best solution at the front. - std::vector<QuerySolution*> solutions; - - // Generate solutions and ranking decisions sorted by score. - for (size_t orderingIndex = 0; - orderingIndex < candidateOrder.size(); ++orderingIndex) { - // index into candidates/ranking - size_t i = candidateOrder[orderingIndex]; - solutions.push_back(_candidates[i].solution); - } - - // Check solution cache data. Do not add to cache if - // we have any invalid SolutionCacheData data. - // XXX: One known example is 2D queries - bool validSolutions = true; - for (size_t i = 0; i < solutions.size(); ++i) { - if (NULL == solutions[i]->cacheData.get()) { - QLOG() << "Not caching query because this solution has no cache data: " - << solutions[i]->toString(); - validSolutions = false; - break; - } - } - - if (validSolutions) { - cache->add(*_query, solutions, _ranking.release()); - } - } - - // Clear out the candidate plans, leaving only stats as we're all done w/them. - // Traverse candidate plans in order or score - for (size_t orderingIndex = 0; - orderingIndex < candidateOrder.size(); ++orderingIndex) { - // index into candidates/ranking - size_t i = candidateOrder[orderingIndex]; - - if (i == _bestChild) { continue; } - if (i == backupChild) { continue; } - - delete _candidates[i].solution; - - // Remember the stats for the candidate plan because we always show it on an - // explain. (The {verbose:false} in explain() is client-side trick; we always - // generate a "verbose" explain.) - PlanStageStats* stats = _candidates[i].root->getStats(); - if (stats) { - _candidateStats.push_back(stats); - } - delete _candidates[i].root; - - // ws must die after the root. - delete _candidates[i].ws; - } - - _candidates.clear(); - } - - bool MultiPlanRunner::hasBackupPlan() const { - return NULL != _backupPlan; - } - - bool MultiPlanRunner::workAllPlans(BSONObj* objOut) { - bool doneWorking = false; - - for (size_t i = 0; i < _candidates.size(); ++i) { - CandidatePlan& candidate = _candidates[i]; - if (candidate.failed) { continue; } - - WorkingSetID id = WorkingSet::INVALID_ID; - PlanStage::StageState state = candidate.root->work(&id); - - if (PlanStage::ADVANCED == state) { - // Save result for later. - candidate.results.push_back(id); - - // Once a plan returns enough results, stop working. - if (candidate.results.size() - >= size_t(internalQueryPlanEvaluationMaxResults)) { - doneWorking = true; - } - } - else if (PlanStage::NEED_TIME == state) { - // Fall through to yield check at end of large conditional. - } - else if (PlanStage::NEED_FETCH == state) { - // id has a loc and refers to an obj we need to fetch. - WorkingSetMember* member = candidate.ws->get(id); - - // This must be true for somebody to request a fetch and can only change when an - // invalidation happens, which is when we give up a lock. Don't give up the - // lock between receiving the NEED_FETCH and actually fetching(?). - verify(member->hasLoc()); - - // Do nothing. TODO: Remove NEED_FETCH entirely from stages. - } - else if (PlanStage::IS_EOF == state) { - // First plan to hit EOF wins automatically. Stop evaluating other plans. - // Assumes that the ranking will pick this plan. - doneWorking = true; - } - else { - // FAILURE or DEAD. Do we want to just tank that plan and try the rest? We - // probably want to fail globally as this shouldn't happen anyway. - - candidate.failed = true; - ++_failureCount; - - // Propage most recent seen failure to parent. - if (PlanStage::FAILURE == state && (NULL != objOut)) { - WorkingSetCommon::getStatusMemberObject(*candidate.ws, id, objOut); - } - - if (_failureCount == _candidates.size()) { - _failure = true; - return false; - } - } - } - - return !doneWorking; - } - - void MultiPlanRunner::allPlansSaveState() { - for (size_t i = 0; i < _candidates.size(); ++i) { - _candidates[i].root->prepareToYield(); - } - } - - void MultiPlanRunner::allPlansRestoreState() { - for (size_t i = 0; i < _candidates.size(); ++i) { - _candidates[i].root->recoverFromYield(); - } - } - - Status MultiPlanRunner::getInfo(TypeExplain** explain, - PlanInfo** planInfo) const { - if (NULL != explain) { - if (NULL == _bestPlan.get()) { - return Status(ErrorCodes::InternalError, "No plan available to provide stats"); - } - - // - // Explain for the winner plan - // - - scoped_ptr<PlanStageStats> stats(_bestPlan->getStats()); - if (NULL == stats.get()) { - return Status(ErrorCodes::InternalError, "no stats available to explain plan"); - } - - return explainMultiPlan(*stats, _candidateStats, _bestSolution.get(), explain); - } - else if (NULL != planInfo) { - if (NULL == _bestSolution.get()) { - return Status(ErrorCodes::InternalError, - "no best solution available for plan info"); - } - getPlanInfo(*_bestSolution, planInfo); - } - - return Status::OK(); - } - -} // namespace mongo diff --git a/src/mongo/db/query/plan_ranker.h b/src/mongo/db/query/plan_ranker.h index 1f0f1779b19..41049febbfc 100644 --- a/src/mongo/db/query/plan_ranker.h +++ b/src/mongo/db/query/plan_ranker.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -83,7 +83,7 @@ namespace mongo { /** * Information about why a plan was picked to be the best. Data here is placed into the cache - * and used by the CachedPlanRunner to compare expected performance with actual. + * and used to compare expected performance with actual. */ struct PlanRankingDecision { /** diff --git a/src/mongo/db/query/stage_builder.cpp b/src/mongo/db/query/stage_builder.cpp index 99bd8bddf9e..47f5b8a92ad 100644 --- a/src/mongo/db/query/stage_builder.cpp +++ b/src/mongo/db/query/stage_builder.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -49,6 +49,7 @@ #include "mongo/db/exec/skip.h" #include "mongo/db/exec/text.h" #include "mongo/db/index/fts_access_method.h" +#include "mongo/db/structure/catalog/namespace_details.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" @@ -315,25 +316,15 @@ namespace mongo { } } - // static + // static (this one is used for Cached and MultiPlanStage) bool StageBuilder::build(Collection* collection, const QuerySolution& solution, - PlanStage** rootOut, - WorkingSet** wsOut) { - QuerySolutionNode* root = solution.root.get(); - if (NULL == root) { return false; } - - auto_ptr<WorkingSet> ws(new WorkingSet()); - PlanStage* stageRoot = buildStages(collection, solution, root, ws.get()); - - if (NULL != stageRoot) { - *rootOut = stageRoot; - *wsOut = ws.release(); - return true; - } - else { - return false; - } + WorkingSet* wsIn, + PlanStage** rootOut) { + if (NULL == wsIn || NULL == rootOut) { return false; } + QuerySolutionNode* solutionNode = solution.root.get(); + if (NULL == solutionNode) { return false; } + return NULL != (*rootOut = buildStages(collection, solution, solutionNode, wsIn)); } } // namespace mongo diff --git a/src/mongo/db/query/stage_builder.h b/src/mongo/db/query/stage_builder.h index 4f3c5be8831..9f3c9f20d60 100644 --- a/src/mongo/db/query/stage_builder.h +++ b/src/mongo/db/query/stage_builder.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -40,18 +40,17 @@ namespace mongo { class StageBuilder { public: /** - * Turns 'solution' into an executable tree of PlanStage(s). This function accesses cc() - * and catalog information and as such the caller must have a lock. + * Turns 'solution' into an executable tree of PlanStage(s). * * Returns true if the PlanStage tree was built successfully. The root of the tree is in - * *rootOut and the WorkingSet that the tree uses is in *wsOut. + * *rootOut and the WorkingSet that the tree uses is in wsIn. * * Returns false otherwise. *rootOut and *wsOut are invalid. */ static bool build(Collection* collection, const QuerySolution& solution, - PlanStage** rootOut, - WorkingSet** wsOut); + WorkingSet* wsIn, + PlanStage** rootOut); }; } // namespace mongo diff --git a/src/mongo/db/query/stage_types.h b/src/mongo/db/query/stage_types.h index cb1ffb391d1..901f804190f 100644 --- a/src/mongo/db/query/stage_types.h +++ b/src/mongo/db/query/stage_types.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -36,6 +36,7 @@ namespace mongo { enum StageType { STAGE_AND_HASH, STAGE_AND_SORTED, + STAGE_CACHED_PLAN, STAGE_COLLSCAN, // If we're running a .count(), the query is fully covered by one ixscan, and the ixscan is @@ -63,6 +64,7 @@ namespace mongo { STAGE_IXSCAN, STAGE_LIMIT, + STAGE_MULTI_PLAN, STAGE_OR, STAGE_PROJECTION, STAGE_SHARDING_FILTER, diff --git a/src/mongo/db/query/subplan_runner.cpp b/src/mongo/db/query/subplan_runner.cpp index da39ec0eb8e..f6400ddd684 100644 --- a/src/mongo/db/query/subplan_runner.cpp +++ b/src/mongo/db/query/subplan_runner.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -32,14 +32,15 @@ #include "mongo/db/catalog/collection.h" #include "mongo/db/diskloc.h" #include "mongo/db/jsobj.h" +#include "mongo/db/exec/multi_plan.h" #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/get_runner.h" -#include "mongo/db/query/multi_plan_runner.h" #include "mongo/db/query/planner_analysis.h" #include "mongo/db/query/planner_access.h" #include "mongo/db/query/qlog.h" #include "mongo/db/query/query_planner.h" #include "mongo/db/query/stage_builder.h" +#include "mongo/db/query/single_solution_runner.h" #include "mongo/db/query/type_explain.h" namespace mongo { @@ -267,13 +268,13 @@ namespace mongo { // 'solutions' is owned by the SubplanRunner instance until // it is popped from the queue. vector<QuerySolution*> solutions = _solutions.front(); + _solutions.pop(); // We already checked for zero solutions in planSubqueries(...). invariant(!solutions.empty()); if (1 == solutions.size()) { // There is only one solution. Transfer ownership to an auto_ptr. - _solutions.pop(); auto_ptr<QuerySolution> autoSoln(solutions[0]); // We want a well-formed *indexed* solution. @@ -303,41 +304,48 @@ namespace mongo { cacheData->children.push_back(autoSoln->cacheData->tree->clone()); } else { - // N solutions, rank them. Takes ownership of safeOrChildCQ. - MultiPlanRunner* mpr = new MultiPlanRunner(_collection, orChildCQ.release()); - - // Dump all the solutions into the MPR. The MPR takes ownership of - // each solution. - _solutions.pop(); - for (size_t i = 0; i < solutions.size(); ++i) { - WorkingSet* ws; - PlanStage* root; - verify(StageBuilder::build(_collection, *solutions[i], &root, &ws)); - // Takes ownership of all arguments. - mpr->addPlan(solutions[i], root, ws); - } + // N solutions, rank them. Takes ownership of orChildCQ. - // Calling pickBestPlan can yield so we must propagate events down to the MPR. - _underlyingRunner.reset(mpr); + // the working set will be shared by the candidate plans and owned by the runner + WorkingSet* sharedWorkingSet = new WorkingSet(); + + MultiPlanStage* multiPlanStage = new MultiPlanStage(_collection, + orChildCQ.get()); + + // Dump all the solutions into the MPR. + for (size_t ix = 0; ix < solutions.size(); ++ix) { + PlanStage* nextPlanRoot; + verify(StageBuilder::build(_collection, + *solutions[ix], + sharedWorkingSet, + &nextPlanRoot)); + + // Owns first two arguments + multiPlanStage->addPlan(solutions[ix], nextPlanRoot, sharedWorkingSet); + } - // Pull out the best plan. - size_t bestPlan; - BSONObj errorObj; - if (!mpr->pickBestPlan(&bestPlan, &errorObj)) { + multiPlanStage->pickBestPlan(); + if (! multiPlanStage->bestPlanChosen()) { QLOG() << "Subplanner: Failed to pick best plan for subchild " - << orChild->toString() - << " error obj is " << errorObj.toString(); + << orChildCQ->toString(); return false; } - // pickBestPlan can yield. Make sure we're not dead any which way. + Runner* mpr = new SingleSolutionRunner(_collection, + orChildCQ.release(), + multiPlanStage->bestSolution(), + multiPlanStage, + sharedWorkingSet); + + _underlyingRunner.reset(mpr); + if (_killed) { QLOG() << "Subplanner: Killed while picking best plan for subchild " << orChild->toString(); return false; } - QuerySolution* bestSoln = solutions[bestPlan]; + QuerySolution* bestSoln = multiPlanStage->bestSolution(); if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) { QLOG() << "Subplanner: No indexed cache data for subchild " @@ -355,7 +363,7 @@ namespace mongo { return false; } - cacheData->children.push_back(solutions[bestPlan]->cacheData->tree->clone()); + cacheData->children.push_back(bestSoln->cacheData->tree->clone()); } } @@ -393,13 +401,24 @@ namespace mongo { // We use one of these even if there is one plan. We do this so that the entry is cached // with stats obtained in the same fashion as a competitive ranking would have obtained // them. - MultiPlanRunner* mpr = new MultiPlanRunner(_collection, _query.release()); - WorkingSet* ws; + MultiPlanStage* multiPlanStage = new MultiPlanStage(_collection, _query.get()); + WorkingSet* ws = new WorkingSet(); PlanStage* root; - verify(StageBuilder::build(_collection, *soln, &root, &ws)); - // Takes ownership of all arguments. - mpr->addPlan(soln, root, ws); + verify(StageBuilder::build(_collection, *soln, ws, &root)); + multiPlanStage->addPlan(soln, root, ws); // Takes ownership first two arguments. + + multiPlanStage->pickBestPlan(); + if (! multiPlanStage->bestPlanChosen()) { + QLOG() << "Subplanner: Failed to pick best plan for subchild " + << _query->toString(); + return false; + } + Runner* mpr = new SingleSolutionRunner(_collection, + _query.release(), + multiPlanStage->bestSolution(), + multiPlanStage, + ws); _underlyingRunner.reset(mpr); return true; diff --git a/src/mongo/dbtests/plan_ranking.cpp b/src/mongo/dbtests/plan_ranking.cpp index e0b9037dbdf..970e59a97ca 100644 --- a/src/mongo/dbtests/plan_ranking.cpp +++ b/src/mongo/dbtests/plan_ranking.cpp @@ -33,10 +33,10 @@ #include "mongo/client/dbclientcursor.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" +#include "mongo/db/exec/multi_plan.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/instance.h" #include "mongo/db/json.h" -#include "mongo/db/query/multi_plan_runner.h" #include "mongo/db/query/get_runner.h" #include "mongo/db/query/qlog.h" #include "mongo/db/query/query_knobs.h" @@ -101,33 +101,32 @@ namespace PlanRankingTests { ASSERT_GREATER_THAN_OR_EQUALS(solutions.size(), 1U); // Fill out the MPR. - _mpr.reset(new MultiPlanRunner(collection, cq)); - + _mps.reset(new MultiPlanStage(collection, cq)); + WorkingSet* ws = new WorkingSet(); // Put each solution from the planner into the MPR. for (size_t i = 0; i < solutions.size(); ++i) { - WorkingSet* ws; PlanStage* root; - ASSERT(StageBuilder::build(collection, *solutions[i], &root, &ws)); + ASSERT(StageBuilder::build(collection, *solutions[i], ws, &root)); // Takes ownership of all arguments. - _mpr->addPlan(solutions[i], root, ws); + _mps->addPlan(solutions[i], root, ws); } - // And return a pointer to the best solution. The MPR owns the pointer. - size_t bestPlan = numeric_limits<size_t>::max(); - BSONObj unused; - ASSERT(_mpr->pickBestPlan(&bestPlan, &unused)); - ASSERT_LESS_THAN(bestPlan, solutions.size()); - // This is what sets a backup plan, should we test for it. - _mpr->cacheBestPlan(); - return solutions[bestPlan]; + _mps->pickBestPlan(); // This is what sets a backup plan, should we test for it. + ASSERT(_mps->bestPlanChosen()); + + size_t bestPlanIdx = _mps->bestPlanIdx(); + ASSERT_LESS_THAN(bestPlanIdx, solutions.size()); + + // And return a pointer to the best solution. + return _mps->bestSolution(); } /** * Was a backup plan picked during the ranking process? */ bool hasBackupPlan() const { - ASSERT(NULL != _mpr.get()); - return _mpr->hasBackupPlan(); + ASSERT(NULL != _mps.get()); + return _mps->hasBackupPlan(); } protected: @@ -138,7 +137,7 @@ namespace PlanRankingTests { private: static DBDirectClient _client; - scoped_ptr<MultiPlanRunner> _mpr; + scoped_ptr<MultiPlanStage> _mps; // Holds the value of global "internalQueryForceIntersectionPlans" setParameter flag. // Restored at end of test invocation regardless of test result. bool _internalQueryForceIntersectionPlans; @@ -707,6 +706,7 @@ namespace PlanRankingTests { // Use index on 'b'. QuerySolution* soln = pickBestPlan(cq); + std::cerr << "PlanRankingWorkPlansLongEnough: soln=" << soln->toString() << std::endl; ASSERT(QueryPlannerTestLib::solutionMatches( "{fetch: {node: {ixscan: {pattern: {b: 1}}}}}", soln->root.get())); diff --git a/src/mongo/dbtests/query_multi_plan_runner.cpp b/src/mongo/dbtests/query_multi_plan_runner.cpp index 77806f66d8b..4efae2be63c 100644 --- a/src/mongo/dbtests/query_multi_plan_runner.cpp +++ b/src/mongo/dbtests/query_multi_plan_runner.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013 10gen Inc. + * Copyright (C) 2013-2014 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -30,11 +30,12 @@ #include "mongo/db/exec/collection_scan.h" #include "mongo/db/exec/fetch.h" #include "mongo/db/exec/index_scan.h" +#include "mongo/db/exec/multi_plan.h" #include "mongo/db/exec/plan_stage.h" +#include "mongo/db/query/single_solution_runner.h" #include "mongo/db/instance.h" #include "mongo/db/json.h" #include "mongo/db/matcher/expression_parser.h" -#include "mongo/db/query/multi_plan_runner.h" #include "mongo/db/catalog/collection.h" #include "mongo/dbtests/dbtests.h" @@ -111,42 +112,50 @@ namespace QueryMultiPlanRunner { const Collection* coll = ctx.ctx().db()->getCollection(ns()); - auto_ptr<WorkingSet> firstWs(new WorkingSet()); - IndexScan* ix = new IndexScan(ixparams, firstWs.get(), NULL); - auto_ptr<PlanStage> firstRoot(new FetchStage(firstWs.get(), ix, NULL, coll)); + auto_ptr<WorkingSet> sharedWs(new WorkingSet()); + IndexScan* ix = new IndexScan(ixparams, sharedWs.get(), NULL); + auto_ptr<PlanStage> firstRoot(new FetchStage(sharedWs.get(), ix, NULL, coll)); // Plan 1: CollScan with matcher. CollectionScanParams csparams; csparams.collection = ctx.ctx().db()->getCollection( ns() ); csparams.direction = CollectionScanParams::FORWARD; - auto_ptr<WorkingSet> secondWs(new WorkingSet()); + // Make the filter. BSONObj filterObj = BSON("foo" << 7); StatusWithMatchExpression swme = MatchExpressionParser::parse(filterObj); verify(swme.isOK()); auto_ptr<MatchExpression> filter(swme.getValue()); // Make the stage. - auto_ptr<PlanStage> secondRoot(new CollectionScan(csparams, secondWs.get(), + auto_ptr<PlanStage> secondRoot(new CollectionScan(csparams, sharedWs.get(), filter.get())); // Hand the plans off to the runner. CanonicalQuery* cq = NULL; verify(CanonicalQuery::canonicalize(ns(), BSON("foo" << 7), &cq).isOK()); verify(NULL != cq); - MultiPlanRunner mpr(coll, cq); - mpr.addPlan(createQuerySolution(), firstRoot.release(), firstWs.release()); - mpr.addPlan(createQuerySolution(), secondRoot.release(), secondWs.release()); + + MultiPlanStage* mps = new MultiPlanStage(ctx.ctx().db()->getCollection(ns()),cq); + mps->addPlan(createQuerySolution(), firstRoot.release(), sharedWs.get()); + mps->addPlan(createQuerySolution(), secondRoot.release(), sharedWs.get()); // Plan 0 aka the first plan aka the index scan should be the best. - size_t best; - BSONObj unused; - ASSERT(mpr.pickBestPlan(&best, &unused)); - ASSERT_EQUALS(size_t(0), best); + mps->pickBestPlan(); + ASSERT(mps->bestPlanChosen()); + ASSERT_EQUALS(0, mps->bestPlanIdx()); + + SingleSolutionRunner sr( + ctx.ctx().db()->getCollection(ns()), + cq, + mps->bestSolution(), + mps, + sharedWs.release() + ); // Get all our results out. int results = 0; BSONObj obj; - while (Runner::RUNNER_ADVANCED == mpr.getNext(&obj, NULL)) { + while (Runner::RUNNER_ADVANCED == sr.getNext(&obj, NULL)) { ASSERT_EQUALS(obj["foo"].numberInt(), 7); ++results; } |