diff options
author | Hari Khalsa <hkhalsa@10gen.com> | 2014-03-11 15:53:55 -0400 |
---|---|---|
committer | Hari Khalsa <hkhalsa@10gen.com> | 2014-03-11 17:47:58 -0400 |
commit | 057a542daf11a62c1f57b3b406cb8fd33804a831 (patch) | |
tree | f649301ef55ed06eedf4c3e067d128b4c3d74c11 /src/mongo | |
parent | 63ec93b3ee18c9c16f8adde07d0e8243911a7108 (diff) | |
download | mongo-057a542daf11a62c1f57b3b406cb8fd33804a831.tar.gz |
SERVER-10026 centralize query heuristic numbers
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/query/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/db/query/get_runner.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/query/multi_plan_runner.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/query/plan_cache.cpp | 24 | ||||
-rw-r--r-- | src/mongo/db/query/plan_cache.h | 20 | ||||
-rw-r--r-- | src/mongo/db/query/plan_cache_test.cpp | 12 | ||||
-rw-r--r-- | src/mongo/db/query/plan_enumerator.h | 6 | ||||
-rw-r--r-- | src/mongo/db/query/plan_ranker.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/query/query_knobs.cpp | 53 | ||||
-rw-r--r-- | src/mongo/db/query/query_knobs.h | 74 | ||||
-rw-r--r-- | src/mongo/db/query/query_planner_params.h | 6 | ||||
-rw-r--r-- | src/mongo/dbtests/plan_ranking.cpp | 14 |
12 files changed, 162 insertions, 66 deletions
diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index 4504fbc3220..29f3ade9520 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -14,6 +14,7 @@ env.Library( "planner_access.cpp", "planner_analysis.cpp", "planner_ixselect.cpp", + "query_knobs.cpp", "query_planner.cpp", "query_solution.cpp", ], @@ -23,6 +24,7 @@ env.Library( "$BUILD_DIR/mongo/bson", "$BUILD_DIR/mongo/expressions", "$BUILD_DIR/mongo/index_names", + "$BUILD_DIR/mongo/server_parameters", ], ) diff --git a/src/mongo/db/query/get_runner.cpp b/src/mongo/db/query/get_runner.cpp index feb769c313c..ea324b4719e 100644 --- a/src/mongo/db/query/get_runner.cpp +++ b/src/mongo/db/query/get_runner.cpp @@ -44,6 +44,7 @@ #include "mongo/db/query/plan_cache.h" #include "mongo/db/query/planner_analysis.h" #include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" #include "mongo/db/query/query_planner.h" #include "mongo/db/query/query_planner_common.h" #include "mongo/db/query/single_solution_runner.h" @@ -55,8 +56,6 @@ namespace mongo { - MONGO_EXPORT_SERVER_PARAMETER(enableIndexIntersection, bool, true); - // static void filterAllowedIndexEntries(const AllowedIndices& allowedIndices, std::vector<IndexEntry>* indexEntries) { @@ -194,7 +193,7 @@ namespace mongo { } } - if (enableIndexIntersection) { + if (internalQueryPlannerEnableIndexIntersection) { plannerParams->options |= QueryPlannerParams::INDEX_INTERSECTION; } diff --git a/src/mongo/db/query/multi_plan_runner.cpp b/src/mongo/db/query/multi_plan_runner.cpp index d55cf34066f..0266cd69a03 100644 --- a/src/mongo/db/query/multi_plan_runner.cpp +++ b/src/mongo/db/query/multi_plan_runner.cpp @@ -41,6 +41,7 @@ #include "mongo/db/query/plan_cache.h" #include "mongo/db/query/plan_executor.h" #include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" #include "mongo/db/query/query_solution.h" #include "mongo/db/query/type_explain.h" #include "mongo/db/catalog/collection.h" @@ -312,10 +313,8 @@ namespace mongo { } bool MultiPlanRunner::pickBestPlan(size_t* out, BSONObj* objOut) { - static const int timesEachPlanIsWorked = 100; - // Run each plan some number of times. - for (int i = 0; i < timesEachPlanIsWorked; ++i) { + for (int i = 0; i < internalQueryPlanEvaluationWorks; ++i) { bool moreToDo = workAllPlans(objOut); if (!moreToDo) { break; } } diff --git a/src/mongo/db/query/plan_cache.cpp b/src/mongo/db/query/plan_cache.cpp index db05a696d63..f192197e200 100644 --- a/src/mongo/db/query/plan_cache.cpp +++ b/src/mongo/db/query/plan_cache.cpp @@ -36,14 +36,11 @@ #include "mongo/db/query/plan_ranker.h" #include "mongo/db/query/query_solution.h" #include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" #include "mongo/util/mongoutils/str.h" namespace mongo { - const int PlanCache::kPlanCacheMaxWriteOperations = 1000; - - const int PlanCache::kMaxCacheSize = 200; - // // Cache-related functions for CanonicalQuery // @@ -189,11 +186,6 @@ namespace mongo { return ss; } - // static - const size_t PlanCacheEntry::kMaxFeedback = 20; - - // static - const double PlanCacheEntry::kStdDevThreshold = 2.0; // static const double PlanCacheEntry::kMinDeviation = 0.0001; @@ -285,9 +277,9 @@ namespace mongo { // PlanCache // - PlanCache::PlanCache() : _cache(kMaxCacheSize) { } + PlanCache::PlanCache() : _cache(internalQueryCacheSize) { } - PlanCache::PlanCache(const std::string& ns) : _cache(kMaxCacheSize), _ns(ns) { } + PlanCache::PlanCache(const std::string& ns) : _cache(internalQueryCacheSize), _ns(ns) { } PlanCache::~PlanCache() { } @@ -399,7 +391,7 @@ namespace mongo { return false; } - if (deviation > (PlanCacheEntry::kStdDevThreshold * (*entry->stddevScore))) { + if (deviation > (internalQueryCacheStdDeviations * (*entry->stddevScore))) { // This run of the plan was much worse than average. // Kick it out of the plan cache. return true; @@ -426,7 +418,7 @@ namespace mongo { } invariant(entry); - if (entry->feedback.size() >= PlanCacheEntry::kMaxFeedback) { + if (entry->feedback.size() >= size_t(internalQueryCacheFeedbacksStored)) { // If we have enough feedback, then use it to determine whether // we should get rid of the cached solution. if (hasCachedPlanPerformanceDegraded(entry, autoFeedback.get())) { @@ -496,10 +488,12 @@ namespace mongo { void PlanCache::notifyOfWriteOp() { // It's fine to clear the cache multiple times if multiple threads // increment the counter to kPlanCacheMaxWriteOperations or greater. - if (_writeOperations.addAndFetch(1) < kPlanCacheMaxWriteOperations) { + if (_writeOperations.addAndFetch(1) < internalQueryCacheWriteOpsBetweenFlush) { return; } - LOG(1) << _ns << ": clearing collection plan cache - " << kPlanCacheMaxWriteOperations + + LOG(1) << _ns << ": clearing collection plan cache - " + << internalQueryCacheWriteOpsBetweenFlush << " write operations detected since last refresh."; clear(); } diff --git a/src/mongo/db/query/plan_cache.h b/src/mongo/db/query/plan_cache.h index 534f0bef1c9..402633c8c5b 100644 --- a/src/mongo/db/query/plan_cache.h +++ b/src/mongo/db/query/plan_cache.h @@ -263,15 +263,6 @@ namespace mongo { // The standard deviation of the scores from stored as feedback. boost::optional<double> stddevScore; - // Determines the amount of feedback that we are willing to store. Must be >= 1. - // TODO: how do we tune this? - static const size_t kMaxFeedback; - - // The number of standard deviations which must be exceeded - // in order to determine that the cache entry should be removed. - // Must be positive. TODO how do we tune this? - static const double kStdDevThreshold; - // In order to justify eviction, the deviation from the mean must exceed a // minimum threshold. static const double kMinDeviation; @@ -288,17 +279,6 @@ namespace mongo { MONGO_DISALLOW_COPYING(PlanCache); public: /** - * Flush cache when the number of write operations since last - * clear() reaches this limit. - */ - static const int kPlanCacheMaxWriteOperations; - - /** - * The maximum number of plan cache entries allowed. - */ - static const int kMaxCacheSize; - - /** * We don't want to cache every possible query. This function * encapsulates the criteria for what makes a canonical query * suitable for lookup/inclusion in the cache. diff --git a/src/mongo/db/query/plan_cache_test.cpp b/src/mongo/db/query/plan_cache_test.cpp index 0f461df186c..15fc5170611 100644 --- a/src/mongo/db/query/plan_cache_test.cpp +++ b/src/mongo/db/query/plan_cache_test.cpp @@ -39,6 +39,7 @@ #include "mongo/db/json.h" #include "mongo/db/query/qlog.h" #include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/query_knobs.h" #include "mongo/db/query/query_planner.h" #include "mongo/db/query/query_planner_test_lib.h" #include "mongo/db/query/query_solution.h" @@ -387,26 +388,25 @@ namespace { ASSERT_OK(planCache.add(*cq, solns, createDecision(1U))); ASSERT_EQUALS(planCache.size(), 1U); - // First (PlanCache::kPlanCacheMaxWriteOperations - 1) notifications should have - // no effect on cache contents. - for (int i = 0; i < (PlanCache::kPlanCacheMaxWriteOperations - 1); ++i) { + // First (N - 1) write ops should have no effect on cache contents. + for (int i = 0; i < (internalQueryCacheWriteOpsBetweenFlush - 1); ++i) { planCache.notifyOfWriteOp(); } ASSERT_EQUALS(planCache.size(), 1U); - // 1000th notification will cause cache to be cleared. + // N-th notification will cause cache to be cleared. planCache.notifyOfWriteOp(); ASSERT_EQUALS(planCache.size(), 0U); // Clearing the cache should reset the internal write // operation counter. - // Repopulate cache. Write (PlanCache::kPlanCacheMaxWriteOperations - 1) times. + // Repopulate cache. Write (N - 1) times. // Clear cache. // Add cache entry again. // After clearing and adding a new entry, the next write operation should not // clear the cache. ASSERT_OK(planCache.add(*cq, solns, createDecision(1U))); - for (int i = 0; i < (PlanCache::kPlanCacheMaxWriteOperations - 1); ++i) { + for (int i = 0; i < (internalQueryCacheWriteOpsBetweenFlush - 1); ++i) { planCache.notifyOfWriteOp(); } ASSERT_EQUALS(planCache.size(), 1U); diff --git a/src/mongo/db/query/plan_enumerator.h b/src/mongo/db/query/plan_enumerator.h index 730d9f2af40..ccbd7e5f373 100644 --- a/src/mongo/db/query/plan_enumerator.h +++ b/src/mongo/db/query/plan_enumerator.h @@ -35,16 +35,14 @@ #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/index_entry.h" #include "mongo/db/query/index_tag.h" +#include "mongo/db/query/query_knobs.h" namespace mongo { struct PlanEnumeratorParams { - // How many choices do we want when computing ixisect solutions in an AND? - static const size_t kDefaultMaxIntersectPerAnd = 3; - PlanEnumeratorParams() : intersect(false), - maxIntersectPerAnd(3) { } + maxIntersectPerAnd(internalQueryEnumerationMaxIntersectPerAnd) { } // Do we provide solutions that use more indices than the minimum required to provide // an indexed solution? diff --git a/src/mongo/db/query/plan_ranker.cpp b/src/mongo/db/query/plan_ranker.cpp index fc2d3b9d4b3..edf16cd9069 100644 --- a/src/mongo/db/query/plan_ranker.cpp +++ b/src/mongo/db/query/plan_ranker.cpp @@ -35,6 +35,7 @@ #include "mongo/db/exec/plan_stage.h" #include "mongo/db/exec/working_set.h" #include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/query_knobs.h" #include "mongo/db/query/query_solution.h" #include "mongo/db/query/qlog.h" #include "mongo/db/server_options.h" @@ -56,8 +57,6 @@ namespace { namespace mongo { - MONGO_EXPORT_SERVER_PARAMETER(forceIntersectionPlans, bool, false); - using std::vector; // static @@ -248,12 +247,12 @@ namespace mongo { QLOG() << scoreStr << endl; LOG(2) << scoreStr; - if (forceIntersectionPlans) { + if (internalQueryForceIntersectionPlans) { if (hasStage(STAGE_AND_HASH, stats) || hasStage(STAGE_AND_SORTED, stats)) { // The boost should be >2.001 to make absolutely sure the ixisect plan will win due // to the combination of 1) productivity, 2) eof bonus, and 3) no ixisect bonus. score += 3; - QLOG() << "Score boosted to " << score << " due to forceIntersectionPlans." << endl; + QLOG() << "Score boosted to " << score << " due to intersection forcing." << endl; } } diff --git a/src/mongo/db/query/query_knobs.cpp b/src/mongo/db/query/query_knobs.cpp new file mode 100644 index 00000000000..3565d5a219c --- /dev/null +++ b/src/mongo/db/query/query_knobs.cpp @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/server_options.h" +#include "mongo/db/server_parameters.h" + +namespace mongo { + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlanEvaluationWorks, int, 100); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheSize, int, 200); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheFeedbacksStored, int, 20); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheStdDeviations, double, 2.0); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheWriteOpsBetweenFlush, int, 1000); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlannerMaxIndexedSolutions, int, 6); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryEnumerationMaxIntersectPerAnd, int, 3); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryForceIntersectionPlans, bool, false); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlannerEnableIndexIntersection, bool, true); + +} // namespace mongo diff --git a/src/mongo/db/query/query_knobs.h b/src/mongo/db/query/query_knobs.h new file mode 100644 index 00000000000..8f703c7d5e8 --- /dev/null +++ b/src/mongo/db/query/query_knobs.h @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { + + // + // multi-plan ranking + // + + // Max number of times we call work() on plans before comparing them. + extern int internalQueryPlanEvaluationWorks; + + // Do we give a big ranking bonus to intersection plans? + extern bool internalQueryForceIntersectionPlans; + + // Do we have ixisect on at all? + extern bool internalQueryPlannerEnableIndexIntersection; + + // + // plan cache + // + + // How many entries in the cache? + extern int internalQueryCacheSize; + + // How many feedback entries do we collect before possibly evicting from the cache based on bad + // performance? + extern int internalQueryCacheFeedbacksStored; + + // How many stddevs must a feedback be from the 'reference' performance for us to evict the + // entry from the cache? + extern double internalQueryCacheStdDeviations; + + // How many write ops should we allow in a collection before tossing all cache entries? + extern int internalQueryCacheWriteOpsBetweenFlush; + + // + // Planning and enumeration. + // + + // How many indexed solutions will QueryPlanner::plan output? + extern int internalQueryPlannerMaxIndexedSolutions; + + // How many intersections will the enumerator consider at each AND? + extern int internalQueryEnumerationMaxIntersectPerAnd; + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner_params.h b/src/mongo/db/query/query_planner_params.h index 14a0d55707a..eacf858b06a 100644 --- a/src/mongo/db/query/query_planner_params.h +++ b/src/mongo/db/query/query_planner_params.h @@ -32,17 +32,15 @@ #include "mongo/db/jsobj.h" #include "mongo/db/query/index_entry.h" +#include "mongo/db/query/query_knobs.h" namespace mongo { struct QueryPlannerParams { - // How many indexed solutions are we willing to output? - static const size_t kDefaultMaxIndexedSolutions = 6; - QueryPlannerParams() : options(DEFAULT), indexFiltersApplied(false), - maxIndexedSolutions(kDefaultMaxIndexedSolutions) { } + maxIndexedSolutions(internalQueryPlannerMaxIndexedSolutions) { } enum Options { // You probably want to set this. diff --git a/src/mongo/dbtests/plan_ranking.cpp b/src/mongo/dbtests/plan_ranking.cpp index 1be5d0ad9d0..7ccd1ce7b71 100644 --- a/src/mongo/dbtests/plan_ranking.cpp +++ b/src/mongo/dbtests/plan_ranking.cpp @@ -48,7 +48,7 @@ namespace mongo { // How we access the external setParameter testing bool. - extern bool forceIntersectionPlans; + extern bool internalQueryForceIntersectionPlans; } // namespace mongo @@ -61,14 +61,14 @@ namespace PlanRankingTests { class PlanRankingTestBase { public: - PlanRankingTestBase() : _forceIntersectionPlans(forceIntersectionPlans) { + PlanRankingTestBase() : _internalQueryForceIntersectionPlans(internalQueryForceIntersectionPlans) { Client::WriteContext ctx(ns); _client.dropCollection(ns); } virtual ~PlanRankingTestBase() { // Restore external setParameter testing bool. - forceIntersectionPlans = _forceIntersectionPlans; + internalQueryForceIntersectionPlans = _internalQueryForceIntersectionPlans; } void insert(const BSONObj& obj) { @@ -148,9 +148,9 @@ namespace PlanRankingTests { private: static DBDirectClient _client; scoped_ptr<MultiPlanRunner> _mpr; - // Holds the value of global "forceIntersectionPlans" setParameter flag. + // Holds the value of global "internalQueryForceIntersectionPlans" setParameter flag. // Restored at end of test invocation regardless of test result. - bool _forceIntersectionPlans; + bool _internalQueryForceIntersectionPlans; }; DBDirectClient PlanRankingTestBase::_client; @@ -186,7 +186,7 @@ namespace PlanRankingTests { // Turn on the "force intersect" option. // This will be reverted by PlanRankingTestBase's destructor when the test completes. - forceIntersectionPlans = true; + internalQueryForceIntersectionPlans = true; // And run the same query again. ASSERT(CanonicalQuery::canonicalize(ns, BSON("a" << 100 << "b" << 1), &cq).isOK()); @@ -229,7 +229,7 @@ namespace PlanRankingTests { // Turn on the "force intersect" option. // This will be reverted by PlanRankingTestBase's destructor when the test completes. - forceIntersectionPlans = true; + internalQueryForceIntersectionPlans = true; // Takes ownership of cq. QuerySolution* soln = pickBestPlan(cq); |