SERVER-62981 Make SBE multi-planner trial period length independent of collection size

This patch changes the 'internalQueryPlanEvaluationCollFraction' knob
to apply only to the classic engine. It introduces a separate knob,
'internalQueryPlanEvaluationCollFractionSbe', which applies only to
the SBE engine. The SBE knob has a default of 0, while the classic
engine retains its default of 0.3. This ensures that by default, no
candidate plan will ever do more than 10,000 storage reads during SBE
multi-planning.

(cherry picked from commit f27f088ecf14825a2ae9cedb2c13093287ded84a)

8 files changed, 132 insertions, 20 deletions

diff --git a/jstests/noPassthrough/query_knobs_validation.js b/jstests/noPassthrough/query_knobs_validation.js
index a647bc24df1..21376673f84 100644
--- a/jstests/noPassthrough/query_knobs_validation.js
+++ b/jstests/noPassthrough/query_knobs_validation.js
@@ -13,6 +13,7 @@ const testDB = conn.getDB("admin");
 const expectedParamDefaults = {
     internalQueryPlanEvaluationWorks: 10000,
     internalQueryPlanEvaluationCollFraction: 0.3,
+    internalQueryPlanEvaluationCollFractionSbe: 0.0,
     internalQueryPlanEvaluationMaxResults: 101,
     internalQueryCacheMaxEntriesPerCollection: 5000,
     // This is a deprecated alias for "internalQueryCacheMaxEntriesPerCollection".
@@ -88,11 +89,14 @@ assertSetParameterSucceeds("internalQueryPlanEvaluationWorks", 11);
 assertSetParameterFails("internalQueryPlanEvaluationWorks", 0);
 assertSetParameterFails("internalQueryPlanEvaluationWorks", -1);
 
-assertSetParameterSucceeds("internalQueryPlanEvaluationCollFraction", 0.0);
-assertSetParameterSucceeds("internalQueryPlanEvaluationCollFraction", 0.444);
-assertSetParameterSucceeds("internalQueryPlanEvaluationCollFraction", 1.0);
-assertSetParameterFails("internalQueryPlanEvaluationCollFraction", -0.1);
-assertSetParameterFails("internalQueryPlanEvaluationCollFraction", 1.0001);
+for (let paramName of ["internalQueryPlanEvaluationCollFraction",
+                       "internalQueryPlanEvaluationCollFractionSbe"]) {
+    assertSetParameterSucceeds(paramName, 0.0);
+    assertSetParameterSucceeds(paramName, 0.444);
+    assertSetParameterSucceeds(paramName, 1.0);
+    assertSetParameterFails(paramName, -0.1);
+    assertSetParameterFails(paramName, 1.0001);
+}
 
 assertSetParameterSucceeds("internalQueryPlanEvaluationMaxResults", 11);
 assertSetParameterSucceeds("internalQueryPlanEvaluationMaxResults", 0);
diff --git a/jstests/noPassthrough/sbe_multiplanner_trial_termination.js b/jstests/noPassthrough/sbe_multiplanner_trial_termination.js
new file mode 100644
index 00000000000..8918df3bdb1
--- /dev/null
+++ b/jstests/noPassthrough/sbe_multiplanner_trial_termination.js
@@ -0,0 +1,91 @@
+/**
+ * Tests the logic around the termination condition for the SBE multiplanner. In particular,
+ * demonstrates that unlike the classic multiplanner, the SBE multiplanner's end condition is by
+ * default not proportional to the size of the collection.
+ */
+(function() {
+"use strict";
+
+const numDocs = 1000;
+const dbName = "sbe_multiplanner_db";
+const collName = "sbe_multiplanner_coll";
+const collFracKnob = "internalQueryPlanEvaluationCollFraction";
+const collFracKnobSbe = "internalQueryPlanEvaluationCollFractionSbe";
+const worksKnob = "internalQueryPlanEvaluationWorks";
+
+const defaultCollFrac = 0.3;
+const trialLengthFromCollFrac = defaultCollFrac * numDocs;
+const trialLengthFromWorksKnob = 0.1 * numDocs;
+
+const conn = MongoRunner.runMongod({});
+assert.neq(conn, null, "mongod failed to start");
+const db = conn.getDB(dbName);
+const coll = db[collName];
+
+// Gets the "allPlansExecution" section from the explain of a query that has zero results, but for
+// which the only two available indexed plans are highly unselective.
+//
+// Also asserts that the explain has the given version number.
+function getAllPlansExecution(explainVersion) {
+    const explain = coll.find({a: 1, b: 1, c: 1}).explain("allPlansExecution");
+    assert.eq(explain.explainVersion, explainVersion, explain);
+    assert(explain.hasOwnProperty("executionStats"), explain);
+    const execStats = explain.executionStats;
+    assert(execStats.hasOwnProperty("allPlansExecution"), explain);
+    return execStats.allPlansExecution;
+}
+
+// Create a collection with two indices, where neither index is helpful in answering the query.
+assert.commandWorked(coll.createIndex({a: 1}));
+assert.commandWorked(coll.createIndex({b: 1}));
+for (let i = 0; i < numDocs; ++i) {
+    assert.commandWorked(coll.insert({a: 1, b: 1}));
+}
+
+// Lower the value of the 'internalQueryPlanEvaluationWorks' so that it is smaller than 30% of the
+// collection. Since the classic multiplanner takes either the works limit or 30% of the collection
+// size -- whichever is larger -- this should cause the trial period to run for about 0.3 * numDocs
+// work cycles.
+const getParamRes = assert.commandWorked(db.adminCommand({getParameter: 1, [collFracKnob]: 1}));
+assert.eq(getParamRes[collFracKnob], defaultCollFrac);
+assert.commandWorked(db.adminCommand({setParameter: 1, [worksKnob]: trialLengthFromWorksKnob}));
+
+// Force the classic engine and run an "allPlansExecution" verbosity explain. Confirm that the trial
+// period terminates based on the the "collection fraction" as opposed to
+// 'internalQueryPlanEvaluationWorks'.
+assert.commandWorked(db.adminCommand({setParameter: 1, internalQueryForceClassicEngine: true}));
+let allPlans = getAllPlansExecution("1");
+for (let plan of allPlans) {
+    assert(plan.hasOwnProperty("executionStages"), plan);
+    const executionStages = plan.executionStages;
+    assert(executionStages.hasOwnProperty("works"), plan);
+    assert.eq(executionStages.works, trialLengthFromCollFrac, plan);
+}
+
+// Verifies that for each SBE plan in the 'allPlans' array, the number of storage reads done by the
+// plan is equal to 'expectedNumReads'.
+function verifySbeNumReads(allPlans, expectedNumReads) {
+    for (let plan of allPlans) {
+        // Infer the number of reads (SBE's equivalent of work units) as the sum of keys and
+        // documents examined.
+        assert(plan.hasOwnProperty("totalKeysExamined"), plan);
+        assert(plan.hasOwnProperty("totalDocsExamined"), plan);
+        const numReads = plan.totalKeysExamined + plan.totalDocsExamined;
+        assert.eq(numReads, expectedNumReads, plan);
+    }
+}
+
+// Allow the query to use SBE. This time, the trial period should terminate based on the works knob.
+assert.commandWorked(db.adminCommand({setParameter: 1, internalQueryForceClassicEngine: false}));
+allPlans = getAllPlansExecution("2");
+verifySbeNumReads(allPlans, trialLengthFromWorksKnob);
+
+// If the SBE "collection fraction" knob is set to the same value as the equivalent knob for the
+// classic engine, then the SBE trial period should now terminate based on this new value of the
+// collection fraction knob.
+assert.commandWorked(db.adminCommand({setParameter: 1, [collFracKnobSbe]: defaultCollFrac}));
+allPlans = getAllPlansExecution("2");
+verifySbeNumReads(allPlans, trialLengthFromCollFrac);
+
+MongoRunner.stopMongod(conn);
+}());
diff --git a/src/mongo/db/exec/multi_plan.cpp b/src/mongo/db/exec/multi_plan.cpp
index 7dfb6ffc5d7..892d05cb5ca 100644
--- a/src/mongo/db/exec/multi_plan.cpp
+++ b/src/mongo/db/exec/multi_plan.cpp
@@ -163,7 +163,8 @@ Status MultiPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) {
     // make sense.
     auto optTimer = getOptTimer();
 
-    size_t numWorks = trial_period::getTrialPeriodMaxWorks(opCtx(), collection());
+    size_t numWorks = trial_period::getTrialPeriodMaxWorks(
+        opCtx(), collection(), internalQueryPlanEvaluationCollFraction.load());
     size_t numResults = trial_period::getTrialPeriodNumToReturn(*_query);
 
     try {
diff --git a/src/mongo/db/exec/trial_period_utils.cpp b/src/mongo/db/exec/trial_period_utils.cpp
index 3ed27ad751c..2472885c7ed 100644
--- a/src/mongo/db/exec/trial_period_utils.cpp
+++ b/src/mongo/db/exec/trial_period_utils.cpp
@@ -34,17 +34,15 @@
 #include "mongo/db/catalog/collection.h"
 
 namespace mongo::trial_period {
-size_t getTrialPeriodMaxWorks(OperationContext* opCtx, const CollectionPtr& collection) {
+size_t getTrialPeriodMaxWorks(OperationContext* opCtx,
+                              const CollectionPtr& collection,
+                              double collFraction) {
     // Run each plan some number of times. This number is at least as great as
     // 'internalQueryPlanEvaluationWorks', but may be larger for big collections.
     size_t numWorks = internalQueryPlanEvaluationWorks.load();
     if (collection) {
-        // For large collections, the number of works is set to be this fraction of the collection
-        // size.
-        double fraction = internalQueryPlanEvaluationCollFraction;
-
         numWorks = std::max(static_cast<size_t>(internalQueryPlanEvaluationWorks.load()),
-                            static_cast<size_t>(fraction * collection->numRecords(opCtx)));
+                            static_cast<size_t>(collFraction * collection->numRecords(opCtx)));
     }
 
     return numWorks;
diff --git a/src/mongo/db/exec/trial_period_utils.h b/src/mongo/db/exec/trial_period_utils.h
index 609d5f3b484..53a9c91a889 100644
--- a/src/mongo/db/exec/trial_period_utils.h
+++ b/src/mongo/db/exec/trial_period_utils.h
@@ -39,9 +39,12 @@ namespace trial_period {
 /**
  * Returns the number of times that we are willing to work a plan during a trial period.
  *
- * Calculated based on a fixed query knob and the size of the collection.
+ * Calculated based on a fixed query knob and the size of the collection multiplied by
+ * 'collFraction'.
  */
-size_t getTrialPeriodMaxWorks(OperationContext* opCtx, const CollectionPtr& collection);
+size_t getTrialPeriodMaxWorks(OperationContext* opCtx,
+                              const CollectionPtr& collection,
+                              double collFraction);
 
 /**
  * Returns the max number of documents which we should allow any plan to return during the
diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl
index 4a23ff8bd0f..cbaa8310d2d 100644
--- a/src/mongo/db/query/query_knobs.idl
+++ b/src/mongo/db/query/query_knobs.idl
@@ -48,7 +48,8 @@ server_parameters:
       gt: 0
 
   internalQueryPlanEvaluationCollFraction:
-    description: "For large collections, the number times we work() candidate plans is taken as this fraction of the collection size."
+    description: "For large collections, the ceiling for the number times we work() candidate plans
+    is taken as this fraction of the collection size. Applies only to the classic execution engine."
     set_at: [ startup, runtime ]
     cpp_varname: "internalQueryPlanEvaluationCollFraction"
     cpp_vartype: AtomicDouble
@@ -57,6 +58,19 @@ server_parameters:
       gte: 0.0
       lte: 1.0
 
+  internalQueryPlanEvaluationCollFractionSbe:
+    description: "For large collections, the ceiling for the number of individual storage cursor
+    reads allowed during the multi-planning trial period is calculated based on this constant.
+    Applies only for for queries using the SBE execution engine. This is the analog of the
+    'internalQueryPlanEvaluationCollFraction' knob above but for SBE."
+    set_at: [ startup, runtime ]
+    cpp_varname: "internalQueryPlanEvaluationCollFractionSbe"
+    cpp_vartype: AtomicDouble
+    default: 0.0
+    validator:
+      gte: 0.0
+      lte: 1.0
+
   internalQueryPlanEvaluationMaxResults:
     description: "Stop working plans once a plan returns this many results."
     set_at: [ startup, runtime ]
diff --git a/src/mongo/db/query/sbe_multi_planner.cpp b/src/mongo/db/query/sbe_multi_planner.cpp
index 9299fda2a02..8f7735e0a39 100644
--- a/src/mongo/db/query/sbe_multi_planner.cpp
+++ b/src/mongo/db/query/sbe_multi_planner.cpp
@@ -46,10 +46,11 @@ namespace mongo::sbe {
 CandidatePlans MultiPlanner::plan(
     std::vector<std::unique_ptr<QuerySolution>> solutions,
     std::vector<std::pair<std::unique_ptr<PlanStage>, stage_builder::PlanStageData>> roots) {
-    auto candidates =
-        collectExecutionStats(std::move(solutions),
-                              std::move(roots),
-                              trial_period::getTrialPeriodMaxWorks(_opCtx, _collection));
+    auto candidates = collectExecutionStats(
+        std::move(solutions),
+        std::move(roots),
+        trial_period::getTrialPeriodMaxWorks(
+            _opCtx, _collection, internalQueryPlanEvaluationCollFractionSbe.load()));
     auto decision = uassertStatusOK(mongo::plan_ranker::pickBestPlan<PlanStageStats>(candidates));
     return finalizeExecutionPlans(std::move(decision), std::move(candidates));
 }
diff --git a/src/mongo/dbtests/plan_ranking.cpp b/src/mongo/dbtests/plan_ranking.cpp
index 9bda5f73ac0..076e0a80abd 100644
--- a/src/mongo/dbtests/plan_ranking.cpp
+++ b/src/mongo/dbtests/plan_ranking.cpp
@@ -214,7 +214,7 @@ public:
         // We get the number of works done during the trial period in order to make sure that there
         // are more documents in the collection than works done in the trial period. This ensures
         // neither of the plans reach EOF or produce results.
-        size_t numWorks = trial_period::getTrialPeriodMaxWorks(opCtx(), nullptr);
+        size_t numWorks = trial_period::getTrialPeriodMaxWorks(opCtx(), nullptr, 0);
         size_t smallNumber = 10;
         // The following condition must be met in order for the following test to work. Specifically
         // this condition guarantees that the score of the plan using the index on d will score