SERVER-10026 centralize query heuristic numbers

11 files changed, 155 insertions, 59 deletions

diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript
index 4504fbc3220..29f3ade9520 100644
--- a/src/mongo/db/query/SConscript
+++ b/src/mongo/db/query/SConscript
@@ -14,6 +14,7 @@ env.Library(
         "planner_access.cpp",
         "planner_analysis.cpp",
         "planner_ixselect.cpp",
+        "query_knobs.cpp",
         "query_planner.cpp",
         "query_solution.cpp",
     ],
@@ -23,6 +24,7 @@ env.Library(
         "$BUILD_DIR/mongo/bson",
         "$BUILD_DIR/mongo/expressions",
         "$BUILD_DIR/mongo/index_names",
+        "$BUILD_DIR/mongo/server_parameters",
     ],
 )
 
diff --git a/src/mongo/db/query/get_runner.cpp b/src/mongo/db/query/get_runner.cpp
index feb769c313c..ea324b4719e 100644
--- a/src/mongo/db/query/get_runner.cpp
+++ b/src/mongo/db/query/get_runner.cpp
@@ -44,6 +44,7 @@
 #include "mongo/db/query/plan_cache.h"
 #include "mongo/db/query/planner_analysis.h"
 #include "mongo/db/query/qlog.h"
+#include "mongo/db/query/query_knobs.h"
 #include "mongo/db/query/query_planner.h"
 #include "mongo/db/query/query_planner_common.h"
 #include "mongo/db/query/single_solution_runner.h"
@@ -55,8 +56,6 @@
 
 namespace mongo {
 
-    MONGO_EXPORT_SERVER_PARAMETER(enableIndexIntersection, bool, true);
-
     // static
     void filterAllowedIndexEntries(const AllowedIndices& allowedIndices,
                                    std::vector<IndexEntry>* indexEntries) {
@@ -194,7 +193,7 @@ namespace mongo {
             }
         }
 
-        if (enableIndexIntersection) {
+        if (internalQueryPlannerEnableIndexIntersection) {
             plannerParams->options |= QueryPlannerParams::INDEX_INTERSECTION;
         }
 
diff --git a/src/mongo/db/query/multi_plan_runner.cpp b/src/mongo/db/query/multi_plan_runner.cpp
index d55cf34066f..0266cd69a03 100644
--- a/src/mongo/db/query/multi_plan_runner.cpp
+++ b/src/mongo/db/query/multi_plan_runner.cpp
@@ -41,6 +41,7 @@
 #include "mongo/db/query/plan_cache.h"
 #include "mongo/db/query/plan_executor.h"
 #include "mongo/db/query/qlog.h"
+#include "mongo/db/query/query_knobs.h"
 #include "mongo/db/query/query_solution.h"
 #include "mongo/db/query/type_explain.h"
 #include "mongo/db/catalog/collection.h"
@@ -312,10 +313,8 @@ namespace mongo {
     }
 
     bool MultiPlanRunner::pickBestPlan(size_t* out, BSONObj* objOut) {
-        static const int timesEachPlanIsWorked = 100;
-
         // Run each plan some number of times.
-        for (int i = 0; i < timesEachPlanIsWorked; ++i) {
+        for (int i = 0; i < internalQueryPlanEvaluationWorks; ++i) {
             bool moreToDo = workAllPlans(objOut);
             if (!moreToDo) { break; }
         }
diff --git a/src/mongo/db/query/plan_cache.cpp b/src/mongo/db/query/plan_cache.cpp
index db05a696d63..f192197e200 100644
--- a/src/mongo/db/query/plan_cache.cpp
+++ b/src/mongo/db/query/plan_cache.cpp
@@ -36,14 +36,11 @@
 #include "mongo/db/query/plan_ranker.h"
 #include "mongo/db/query/query_solution.h"
 #include "mongo/db/query/qlog.h"
+#include "mongo/db/query/query_knobs.h"
 #include "mongo/util/mongoutils/str.h"
 
 namespace mongo {
 
-    const int PlanCache::kPlanCacheMaxWriteOperations = 1000;
-
-    const int PlanCache::kMaxCacheSize = 200;
-
     //
     // Cache-related functions for CanonicalQuery
     //
@@ -189,11 +186,6 @@ namespace mongo {
         return ss;
     }
 
-    // static
-    const size_t PlanCacheEntry::kMaxFeedback = 20;
-
-    // static
-    const double PlanCacheEntry::kStdDevThreshold = 2.0;
 
     // static
     const double PlanCacheEntry::kMinDeviation = 0.0001;
@@ -285,9 +277,9 @@ namespace mongo {
     // PlanCache
     //
 
-    PlanCache::PlanCache() : _cache(kMaxCacheSize) { }
+    PlanCache::PlanCache() : _cache(internalQueryCacheSize) { }
 
-    PlanCache::PlanCache(const std::string& ns) : _cache(kMaxCacheSize), _ns(ns) { }
+    PlanCache::PlanCache(const std::string& ns) : _cache(internalQueryCacheSize), _ns(ns) { }
 
     PlanCache::~PlanCache() { }
 
@@ -399,7 +391,7 @@ namespace mongo {
             return false;
         }
 
-        if (deviation > (PlanCacheEntry::kStdDevThreshold * (*entry->stddevScore))) {
+        if (deviation > (internalQueryCacheStdDeviations * (*entry->stddevScore))) {
             // This run of the plan was much worse than average.
             // Kick it out of the plan cache.
             return true;
@@ -426,7 +418,7 @@ namespace mongo {
         }
         invariant(entry);
 
-        if (entry->feedback.size() >= PlanCacheEntry::kMaxFeedback) {
+        if (entry->feedback.size() >= size_t(internalQueryCacheFeedbacksStored)) {
             // If we have enough feedback, then use it to determine whether
             // we should get rid of the cached solution.
             if (hasCachedPlanPerformanceDegraded(entry, autoFeedback.get())) {
@@ -496,10 +488,12 @@ namespace mongo {
     void PlanCache::notifyOfWriteOp() {
         // It's fine to clear the cache multiple times if multiple threads
         // increment the counter to kPlanCacheMaxWriteOperations or greater.
-        if (_writeOperations.addAndFetch(1) < kPlanCacheMaxWriteOperations) {
+        if (_writeOperations.addAndFetch(1) < internalQueryCacheWriteOpsBetweenFlush) {
             return;
         }
-        LOG(1) << _ns << ": clearing collection plan cache - " << kPlanCacheMaxWriteOperations
+
+        LOG(1) << _ns << ": clearing collection plan cache - "
+               << internalQueryCacheWriteOpsBetweenFlush
                << " write operations detected since last refresh.";
         clear();
     }
diff --git a/src/mongo/db/query/plan_cache.h b/src/mongo/db/query/plan_cache.h
index 534f0bef1c9..402633c8c5b 100644
--- a/src/mongo/db/query/plan_cache.h
+++ b/src/mongo/db/query/plan_cache.h
@@ -263,15 +263,6 @@ namespace mongo {
         // The standard deviation of the scores from stored as feedback.
         boost::optional<double> stddevScore;
 
-        // Determines the amount of feedback that we are willing to store. Must be >= 1.
-        // TODO: how do we tune this?
-        static const size_t kMaxFeedback;
-
-        // The number of standard deviations which must be exceeded
-        // in order to determine that the cache entry should be removed.
-        // Must be positive. TODO how do we tune this?
-        static const double kStdDevThreshold;
-
         // In order to justify eviction, the deviation from the mean must exceed a
         // minimum threshold.
         static const double kMinDeviation;
@@ -288,17 +279,6 @@ namespace mongo {
         MONGO_DISALLOW_COPYING(PlanCache);
     public:
         /**
-         * Flush cache when the number of write operations since last
-         * clear() reaches this limit.
-         */
-        static const int kPlanCacheMaxWriteOperations;
-
-        /**
-         * The maximum number of plan cache entries allowed.
-         */
-        static const int kMaxCacheSize;
-
-        /**
          * We don't want to cache every possible query. This function
          * encapsulates the criteria for what makes a canonical query
          * suitable for lookup/inclusion in the cache.
diff --git a/src/mongo/db/query/plan_cache_test.cpp b/src/mongo/db/query/plan_cache_test.cpp
index 0f461df186c..15fc5170611 100644
--- a/src/mongo/db/query/plan_cache_test.cpp
+++ b/src/mongo/db/query/plan_cache_test.cpp
@@ -39,6 +39,7 @@
 #include "mongo/db/json.h"
 #include "mongo/db/query/qlog.h"
 #include "mongo/db/query/plan_ranker.h"
+#include "mongo/db/query/query_knobs.h"
 #include "mongo/db/query/query_planner.h"
 #include "mongo/db/query/query_planner_test_lib.h"
 #include "mongo/db/query/query_solution.h"
@@ -387,26 +388,25 @@ namespace {
         ASSERT_OK(planCache.add(*cq, solns, createDecision(1U)));
         ASSERT_EQUALS(planCache.size(), 1U);
 
-        // First (PlanCache::kPlanCacheMaxWriteOperations - 1) notifications should have
-        // no effect on cache contents.
-        for (int i = 0; i < (PlanCache::kPlanCacheMaxWriteOperations - 1); ++i) {
+        // First (N - 1) write ops should have no effect on cache contents.
+        for (int i = 0; i < (internalQueryCacheWriteOpsBetweenFlush - 1); ++i) {
             planCache.notifyOfWriteOp();
         }
         ASSERT_EQUALS(planCache.size(), 1U);
 
-        // 1000th notification will cause cache to be cleared.
+        // N-th notification will cause cache to be cleared.
         planCache.notifyOfWriteOp();
         ASSERT_EQUALS(planCache.size(), 0U);
 
         // Clearing the cache should reset the internal write
         // operation counter.
-        // Repopulate cache. Write (PlanCache::kPlanCacheMaxWriteOperations - 1) times.
+        // Repopulate cache. Write (N - 1) times.
         // Clear cache.
         // Add cache entry again.
         // After clearing and adding a new entry, the next write operation should not
         // clear the cache.
         ASSERT_OK(planCache.add(*cq, solns, createDecision(1U)));
-        for (int i = 0; i < (PlanCache::kPlanCacheMaxWriteOperations - 1); ++i) {
+        for (int i = 0; i < (internalQueryCacheWriteOpsBetweenFlush - 1); ++i) {
             planCache.notifyOfWriteOp();
         }
         ASSERT_EQUALS(planCache.size(), 1U);
diff --git a/src/mongo/db/query/plan_enumerator.h b/src/mongo/db/query/plan_enumerator.h
index 730d9f2af40..ccbd7e5f373 100644
--- a/src/mongo/db/query/plan_enumerator.h
+++ b/src/mongo/db/query/plan_enumerator.h
@@ -35,16 +35,14 @@
 #include "mongo/db/query/canonical_query.h"
 #include "mongo/db/query/index_entry.h"
 #include "mongo/db/query/index_tag.h"
+#include "mongo/db/query/query_knobs.h"
 
 namespace mongo {
 
     struct PlanEnumeratorParams {
 
-        // How many choices do we want when computing ixisect solutions in an AND?
-        static const size_t kDefaultMaxIntersectPerAnd = 3;
-
         PlanEnumeratorParams() : intersect(false),
-                                 maxIntersectPerAnd(3) { }
+                                 maxIntersectPerAnd(internalQueryEnumerationMaxIntersectPerAnd) { }
 
         // Do we provide solutions that use more indices than the minimum required to provide
         // an indexed solution?
diff --git a/src/mongo/db/query/plan_ranker.cpp b/src/mongo/db/query/plan_ranker.cpp
index fc2d3b9d4b3..edf16cd9069 100644
--- a/src/mongo/db/query/plan_ranker.cpp
+++ b/src/mongo/db/query/plan_ranker.cpp
@@ -35,6 +35,7 @@
 #include "mongo/db/exec/plan_stage.h"
 #include "mongo/db/exec/working_set.h"
 #include "mongo/db/query/explain_plan.h"
+#include "mongo/db/query/query_knobs.h"
 #include "mongo/db/query/query_solution.h"
 #include "mongo/db/query/qlog.h"
 #include "mongo/db/server_options.h"
@@ -56,8 +57,6 @@ namespace {
 
 namespace mongo {
 
-    MONGO_EXPORT_SERVER_PARAMETER(forceIntersectionPlans, bool, false);
-
     using std::vector;
 
     // static 
@@ -248,12 +247,12 @@ namespace mongo {
         QLOG() << scoreStr << endl;
         LOG(2) << scoreStr;
 
-        if (forceIntersectionPlans) {
+        if (internalQueryForceIntersectionPlans) {
             if (hasStage(STAGE_AND_HASH, stats) || hasStage(STAGE_AND_SORTED, stats)) {
                 // The boost should be >2.001 to make absolutely sure the ixisect plan will win due
                 // to the combination of 1) productivity, 2) eof bonus, and 3) no ixisect bonus.
                 score += 3;
-                QLOG() << "Score boosted to " << score << " due to forceIntersectionPlans." << endl;
+                QLOG() << "Score boosted to " << score << " due to intersection forcing." << endl;
             }
         }
 
diff --git a/src/mongo/db/query/query_knobs.cpp b/src/mongo/db/query/query_knobs.cpp
new file mode 100644
index 00000000000..3565d5a219c
--- /dev/null
+++ b/src/mongo/db/query/query_knobs.cpp
@@ -0,0 +1,53 @@
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/query/query_knobs.h"
+#include "mongo/db/server_options.h"
+#include "mongo/db/server_parameters.h"
+
+namespace mongo {
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlanEvaluationWorks, int, 100);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheSize, int, 200);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheFeedbacksStored, int, 20);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheStdDeviations, double, 2.0);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheWriteOpsBetweenFlush, int, 1000);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlannerMaxIndexedSolutions, int, 6);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryEnumerationMaxIntersectPerAnd, int, 3);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryForceIntersectionPlans, bool, false);
+
+    MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlannerEnableIndexIntersection, bool, true);
+
+}  // namespace mongo
diff --git a/src/mongo/db/query/query_knobs.h b/src/mongo/db/query/query_knobs.h
new file mode 100644
index 00000000000..8f703c7d5e8
--- /dev/null
+++ b/src/mongo/db/query/query_knobs.h
@@ -0,0 +1,74 @@
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+namespace mongo {
+
+    //
+    // multi-plan ranking
+    //
+
+    // Max number of times we call work() on plans before comparing them.
+    extern int internalQueryPlanEvaluationWorks;
+
+    // Do we give a big ranking bonus to intersection plans?
+    extern bool internalQueryForceIntersectionPlans;
+
+    // Do we have ixisect on at all?
+    extern bool internalQueryPlannerEnableIndexIntersection;
+
+    //
+    // plan cache
+    //
+
+    // How many entries in the cache?
+    extern int internalQueryCacheSize;
+
+    // How many feedback entries do we collect before possibly evicting from the cache based on bad
+    // performance?
+    extern int internalQueryCacheFeedbacksStored;
+
+    // How many stddevs must a feedback be from the 'reference' performance for us to evict the
+    // entry from the cache?
+    extern double internalQueryCacheStdDeviations;
+
+    // How many write ops should we allow in a collection before tossing all cache entries?
+    extern int internalQueryCacheWriteOpsBetweenFlush;
+
+    //
+    // Planning and enumeration.
+    //
+
+    // How many indexed solutions will QueryPlanner::plan output?
+    extern int internalQueryPlannerMaxIndexedSolutions;
+
+    // How many intersections will the enumerator consider at each AND?
+    extern int internalQueryEnumerationMaxIntersectPerAnd;
+
+}  // namespace mongo
diff --git a/src/mongo/db/query/query_planner_params.h b/src/mongo/db/query/query_planner_params.h
index 14a0d55707a..eacf858b06a 100644
--- a/src/mongo/db/query/query_planner_params.h
+++ b/src/mongo/db/query/query_planner_params.h
@@ -32,17 +32,15 @@
 
 #include "mongo/db/jsobj.h"
 #include "mongo/db/query/index_entry.h"
+#include "mongo/db/query/query_knobs.h"
 
 namespace mongo {
 
     struct QueryPlannerParams {
 
-        // How many indexed solutions are we willing to output?
-        static const size_t kDefaultMaxIndexedSolutions = 6;
-
         QueryPlannerParams() : options(DEFAULT),
                                indexFiltersApplied(false),
-                               maxIndexedSolutions(kDefaultMaxIndexedSolutions) { }
+                               maxIndexedSolutions(internalQueryPlannerMaxIndexedSolutions) { }
 
         enum Options {
             // You probably want to set this.