SERVER-59681 Start storing SBE plans in the new plan cache

25 files changed, 192 insertions, 2 deletions

diff --git a/jstests/core/collation_plan_cache.js b/jstests/core/collation_plan_cache.js
index 50e6921c91b..e9c25a36280 100644
--- a/jstests/core/collation_plan_cache.js
+++ b/jstests/core/collation_plan_cache.js
@@ -11,6 +11,12 @@
 // ]
 (function() {
 'use strict';
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 
 var coll = db.collation_plan_cache;
 coll.drop();
diff --git a/jstests/core/index_filter_commands.js b/jstests/core/index_filter_commands.js
index 25ff7fc39ac..0a27a1ae5e1 100644
--- a/jstests/core/index_filter_commands.js
+++ b/jstests/core/index_filter_commands.js
@@ -39,6 +39,10 @@ load("jstests/libs/fixture_helpers.js");      // For 'FixtureHelpers'.
 load("jstests/libs/sbe_explain_helpers.js");  // For 'assertIdHackPlan()'.
 load("jstests/libs/sbe_util.js");             // For checkSBEEnabled.
 
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 const coll = db.jstests_index_filter_commands;
 
 coll.drop();
diff --git a/jstests/core/introspect_hidden_index_plan_cache_entries.js b/jstests/core/introspect_hidden_index_plan_cache_entries.js
index 91645f46b5b..1af13d32d41 100644
--- a/jstests/core/introspect_hidden_index_plan_cache_entries.js
+++ b/jstests/core/introspect_hidden_index_plan_cache_entries.js
@@ -13,6 +13,12 @@
 
 (function() {
 'use strict';
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 
 const collName = 'introspect_hidden_index_plan_cache_entries';
 const collNotAffectedName = 'introspect_hidden_index_plan_cache_entries_unaffected';
diff --git a/jstests/core/plan_cache_clear.js b/jstests/core/plan_cache_clear.js
index f88e170bec6..02eed91081c 100644
--- a/jstests/core/plan_cache_clear.js
+++ b/jstests/core/plan_cache_clear.js
@@ -14,6 +14,14 @@
 // ]
 
 (function() {
+'use strict';
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
+
 const coll = db.jstests_plan_cache_clear;
 coll.drop();
 
diff --git a/jstests/core/plan_cache_list_plans.js b/jstests/core/plan_cache_list_plans.js
index 13c28cfb1a8..c098acc6ee5 100644
--- a/jstests/core/plan_cache_list_plans.js
+++ b/jstests/core/plan_cache_list_plans.js
@@ -16,6 +16,13 @@
 
 (function() {
 "use strict";
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
+
 let coll = db.jstests_plan_cache_list_plans;
 coll.drop();
 
diff --git a/jstests/core/plan_cache_list_shapes.js b/jstests/core/plan_cache_list_shapes.js
index 66997c95ded..8e6a86b1494 100644
--- a/jstests/core/plan_cache_list_shapes.js
+++ b/jstests/core/plan_cache_list_shapes.js
@@ -13,6 +13,14 @@
 //   assumes_unsharded_collection,
 // ]
 (function() {
+'use strict';
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
+
 const coll = db.jstests_plan_cache_list_shapes;
 coll.drop();
 
diff --git a/jstests/core/plan_cache_sbe.js b/jstests/core/plan_cache_sbe.js
index 6069dae2e25..2cd26803858 100644
--- a/jstests/core/plan_cache_sbe.js
+++ b/jstests/core/plan_cache_sbe.js
@@ -19,6 +19,11 @@
 
 load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
 
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
+
 const coll = db.plan_cache_sbe;
 coll.drop();
 
diff --git a/jstests/core/plan_cache_shell_helpers.js b/jstests/core/plan_cache_shell_helpers.js
index aba0d20047e..5bd809988cb 100644
--- a/jstests/core/plan_cache_shell_helpers.js
+++ b/jstests/core/plan_cache_shell_helpers.js
@@ -12,6 +12,14 @@
 //   does_not_support_stepdowns,
 // ]
 (function() {
+'use strict';
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
+
 var coll = db.jstests_plan_cache_shell_helpers;
 coll.drop();
 
diff --git a/jstests/core/plan_cache_stats_shard_and_host.js b/jstests/core/plan_cache_stats_shard_and_host.js
index e81dcd60692..a6bc62e13db 100644
--- a/jstests/core/plan_cache_stats_shard_and_host.js
+++ b/jstests/core/plan_cache_stats_shard_and_host.js
@@ -11,6 +11,12 @@
 "use strict";
 
 load("jstests/libs/fixture_helpers.js");  // For 'FixtureHelpers'.
+load("jstests/libs/sbe_util.js");         // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 
 const coll = db.plan_cache_stats_shard_and_host;
 coll.drop();
diff --git a/jstests/core/profile_find.js b/jstests/core/profile_find.js
index 05d9ee3a757..d69a2ce5754 100644
--- a/jstests/core/profile_find.js
+++ b/jstests/core/profile_find.js
@@ -10,6 +10,12 @@
 
 // For 'getLatestProfilerEntry()'.
 load("jstests/libs/profiler.js");
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 
 var testDB = db.getSiblingDB("profile_find");
 assert.commandWorked(testDB.dropDatabase());
diff --git a/jstests/core/profile_query_hash.js b/jstests/core/profile_query_hash.js
index 51b00d55996..6b88cbf13b8 100644
--- a/jstests/core/profile_query_hash.js
+++ b/jstests/core/profile_query_hash.js
@@ -11,6 +11,12 @@
 
 // For getLatestProfilerEntry
 load("jstests/libs/profiler.js");
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 
 const testDB = db.getSiblingDB("query_hash");
 assert.commandWorked(testDB.dropDatabase());
diff --git a/jstests/core/wildcard_index_cached_plans.js b/jstests/core/wildcard_index_cached_plans.js
index 6e98199dbe0..50d97aef1ad 100644
--- a/jstests/core/wildcard_index_cached_plans.js
+++ b/jstests/core/wildcard_index_cached_plans.js
@@ -22,6 +22,11 @@ load("jstests/libs/collection_drop_recreate.js");  // For assert[Drop|Create]Col
 load('jstests/libs/fixture_helpers.js');  // For getPrimaryForNodeHostingDatabase and isMongos.
 load("jstests/libs/sbe_util.js");         // For checkSBEEnabled.
 
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
+
 const coll = db.wildcard_cached_plans;
 coll.drop();
 
diff --git a/jstests/libs/sbe_util.js b/jstests/libs/sbe_util.js
index a47aa3a1130..bb9bdd0d1b7 100644
--- a/jstests/libs/sbe_util.js
+++ b/jstests/libs/sbe_util.js
@@ -8,8 +8,9 @@ load("jstests/libs/fixture_helpers.js");    // For 'isMongos'
 /**
  * Returns whether or not SBE is enabled for the given connection. Assumes that for repl sets and
  * sharded clusters, SBE is either enabled on each node, or disabled on each node.
+ * If 'featureFlags' is non-empty, checks if SBE and all the feature flags are enabled.
  */
-function checkSBEEnabled(theDB) {
+function checkSBEEnabled(theDB, featureFlags = []) {
     let checkResult = true;
 
     assert.soon(() => {
@@ -32,6 +33,12 @@ function checkSBEEnabled(theDB) {
                     continue;
                 }
 
+                featureFlags.forEach(function(featureFlag) {
+                    const featureFlagParam = conn.adminCommand({getParameter: 1, [featureFlag]: 1});
+                    checkResult = checkResult && featureFlagParam.hasOwnProperty(featureFlag) &&
+                        featureFlagParam[featureFlag]["value"];
+                });
+
                 const getParam =
                     conn.adminCommand({getParameter: 1, internalQueryForceClassicEngine: 1});
                 if (getParam.hasOwnProperty("internalQueryForceClassicEngine") &&
diff --git a/jstests/noPassthrough/log_and_profile_query_hash.js b/jstests/noPassthrough/log_and_profile_query_hash.js
index e4344eee337..5da528a8fef 100644
--- a/jstests/noPassthrough/log_and_profile_query_hash.js
+++ b/jstests/noPassthrough/log_and_profile_query_hash.js
@@ -7,6 +7,7 @@
 // For getLatestProfilerEntry().
 load("jstests/libs/profiler.js");
 load("jstests/libs/logv2_helpers.js");
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
 
 // Prevent the mongo shell from gossiping its cluster time, since this will increase the amount
 // of data logged for each op. For some of the testcases below, including the cluster time would
@@ -19,6 +20,12 @@ assert.neq(null, conn, "mongod was unable to start up");
 const testDB = conn.getDB("jstests_query_shape_hash");
 const coll = testDB.test;
 
+if (checkSBEEnabled(testDB, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
 const profileEntryFilter = {
     op: "query"
 };
diff --git a/jstests/noPassthrough/plan_cache_index_create.js b/jstests/noPassthrough/plan_cache_index_create.js
index 7e9bb8efd89..3ce1731b847 100644
--- a/jstests/noPassthrough/plan_cache_index_create.js
+++ b/jstests/noPassthrough/plan_cache_index_create.js
@@ -9,6 +9,7 @@
 "use strict";
 
 load('jstests/libs/analyze_plan.js');  // For getCachedPlan().
+load("jstests/libs/sbe_util.js");      // For checkSBEEnabled.
 
 const dbName = "test";
 const collName = "coll";
@@ -177,6 +178,12 @@ rst.initiate();
 const primaryDB = rst.getPrimary().getDB(dbName);
 const secondaryDB = rst.getSecondary().getDB(dbName);
 
+if (checkSBEEnabled(primaryDB, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    rst.stopSet();
+    return;
+}
+
 runTest({rst: rst, readDB: primaryDB, writeDB: primaryDB});
 runTest({rst: rst, readDB: secondaryDB, writeDB: primaryDB});
 
diff --git a/jstests/noPassthrough/plan_cache_list_failed_plans.js b/jstests/noPassthrough/plan_cache_list_failed_plans.js
index 2fa5b641b2f..e1fb5161c71 100644
--- a/jstests/noPassthrough/plan_cache_list_failed_plans.js
+++ b/jstests/noPassthrough/plan_cache_list_failed_plans.js
@@ -10,6 +10,12 @@ const testDB = conn.getDB("jstests_plan_cache_list_failed_plans");
 const coll = testDB.test;
 const isSBEEnabled = checkSBEEnabled(testDB);
 
+if (checkSBEEnabled(testDB, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
 coll.drop();
 
 // Setup the database such that it will generate a failing plan and a succeeding plan.
diff --git a/jstests/noPassthrough/plan_cache_memory_debug_info.js b/jstests/noPassthrough/plan_cache_memory_debug_info.js
index a27b639665b..60406402158 100644
--- a/jstests/noPassthrough/plan_cache_memory_debug_info.js
+++ b/jstests/noPassthrough/plan_cache_memory_debug_info.js
@@ -4,6 +4,7 @@
  */
 (function() {
 "use strict";
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
 
 /**
  * Creates two indexes for the given collection. In order for plans to be cached, there need to be
@@ -75,6 +76,12 @@ const conn = MongoRunner.runMongod({});
 assert.neq(conn, null, "mongod failed to start");
 const db = conn.getDB("test");
 const coll = db.plan_cache_memory_debug_info;
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    MongoRunner.stopMongod(conn);
+    return;
+}
 coll.drop();
 createIndexesForColl(coll);
 
diff --git a/jstests/noPassthrough/plan_cache_metrics.js b/jstests/noPassthrough/plan_cache_metrics.js
index 0ef39bfd753..21a114bf12c 100644
--- a/jstests/noPassthrough/plan_cache_metrics.js
+++ b/jstests/noPassthrough/plan_cache_metrics.js
@@ -4,6 +4,8 @@
  */
 (function() {
 "use strict";
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
 const conn = MongoRunner.runMongod({});
 const db = conn.getDB('test');
 const coll1 = db.query_metrics1;
@@ -11,6 +13,12 @@ const coll2 = db.query_metrics2;
 coll1.drop();
 coll2.drop();
 
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
 const queryObj = {
     a: {$gte: 99},
     b: -1
diff --git a/jstests/noPassthrough/plan_cache_replan_sort.js b/jstests/noPassthrough/plan_cache_replan_sort.js
index 04eacc37873..6683b502592 100644
--- a/jstests/noPassthrough/plan_cache_replan_sort.js
+++ b/jstests/noPassthrough/plan_cache_replan_sort.js
@@ -10,12 +10,19 @@
 
 load("jstests/libs/analyze_plan.js");
 load("jstests/libs/profiler.js");
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
 
 const conn = MongoRunner.runMongod();
 const db = conn.getDB("test");
 const coll = db.plan_cache_replan_sort;
 coll.drop();
 
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
 // Ensure a plan with a sort stage gets cached.
 assert.commandWorked(coll.createIndex({x: 1}));
 assert.commandWorked(coll.createIndex({y: 1}));
diff --git a/jstests/noPassthrough/plan_cache_stats_agg_source.js b/jstests/noPassthrough/plan_cache_stats_agg_source.js
index c34eb879b49..cedec94cd3f 100644
--- a/jstests/noPassthrough/plan_cache_stats_agg_source.js
+++ b/jstests/noPassthrough/plan_cache_stats_agg_source.js
@@ -13,6 +13,12 @@ assert.neq(null, conn, "mongod failed to start up");
 const testDb = conn.getDB("test");
 const coll = testDb.plan_cache_stats_agg_source;
 
+if (checkSBEEnabled(testDb, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
 // Note that the "getParameter" command is expected to fail in versions of mongod that do not yet
 // include the slot-based execution engine. When that happens, however, 'isSBEEnabled' still
 // correctly evaluates to false.
diff --git a/jstests/noPassthroughWithMongod/ne_array_indexability.js b/jstests/noPassthroughWithMongod/ne_array_indexability.js
index 606956c5753..8c88f09db8e 100644
--- a/jstests/noPassthroughWithMongod/ne_array_indexability.js
+++ b/jstests/noPassthroughWithMongod/ne_array_indexability.js
@@ -2,9 +2,16 @@
  * Test that $ne: [] queries are cached correctly. See SERVER-39764.
  */
 (function() {
+load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
+
 const coll = db.ne_array_indexability;
 coll.drop();
 
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
+
 coll.createIndex({"obj": 1});
 coll.createIndex({"obj": 1, "abc": 1});
 
diff --git a/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js b/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js
index 8a39fda391e..aec262b01e2 100644
--- a/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js
+++ b/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js
@@ -6,6 +6,12 @@
 "use strict";
 
 load('jstests/libs/analyze_plan.js');  // For isCollScan.
+load("jstests/libs/sbe_util.js");      // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 
 const coll = db.plan_cache_not_in_regex;
 coll.drop();
diff --git a/jstests/noPassthroughWithMongod/plan_cache_replanning.js b/jstests/noPassthroughWithMongod/plan_cache_replanning.js
index be6a1c48f7b..c2043833c9d 100644
--- a/jstests/noPassthroughWithMongod/plan_cache_replanning.js
+++ b/jstests/noPassthroughWithMongod/plan_cache_replanning.js
@@ -8,6 +8,12 @@
 
 load('jstests/libs/analyze_plan.js');              // For getPlanStage().
 load("jstests/libs/collection_drop_recreate.js");  // For assert[Drop|Create]Collection.
+load("jstests/libs/sbe_util.js");                  // For checkSBEEnabled.
+
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+    jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
+    return;
+}
 
 let coll = assertDropAndRecreateCollection(db, "plan_cache_replanning");
 
diff --git a/src/mongo/db/exec/plan_cache_util.h b/src/mongo/db/exec/plan_cache_util.h
index 5ef7cd8fd45..a03ac6a9eb2 100644
--- a/src/mongo/db/exec/plan_cache_util.h
+++ b/src/mongo/db/exec/plan_cache_util.h
@@ -34,6 +34,7 @@
 #include "mongo/db/query/collection_query_info.h"
 #include "mongo/db/query/plan_cache_key_factory.h"
 #include "mongo/db/query/plan_explainer_factory.h"
+#include "mongo/db/query/sbe_plan_cache.h"
 #include "mongo/db/query/sbe_plan_ranker.h"
 
 namespace mongo {
@@ -181,7 +182,7 @@ void updatePlanCache(
             }
         }
 
-        if (validSolutions) {
+        auto cacheClassicPlan = [&]() {
             // The caller of this constructor is responsible for ensuring that the QuerySolution
             // has valid cacheData. If there's no data to cache you shouldn't be trying to
             // construct a PlanCacheEntry.
@@ -202,6 +203,34 @@ void updatePlanCache(
                                       opCtx->getServiceContext()->getPreciseClockSource()->now(),
                                       boost::none, /* worksGrowthCoefficient */
                                       &callbacks));
+        };
+
+        if (validSolutions) {
+            if constexpr (std::is_same_v<PlanStageType, std::unique_ptr<sbe::PlanStage>>) {
+                if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) {
+                    // Clone the winning SBE plan and its auxiliary data.
+                    auto cachedPlan = std::make_unique<sbe::CachedSbePlan>(
+                        candidates[winnerIdx].root->clone(), candidates[winnerIdx].data);
+
+                    PlanCacheLoggingCallbacks<sbe::PlanCacheKey, sbe::CachedSbePlan> callbacks{
+                        query};
+                    uassertStatusOK(sbe::getPlanCache(opCtx).set(
+                        plan_cache_key_factory::make<sbe::PlanCacheKey>(query, collection),
+                        std::move(cachedPlan),
+                        solutions,
+                        std::move(ranking),
+                        opCtx->getServiceContext()->getPreciseClockSource()->now(),
+                        boost::none, /* worksGrowthCoefficient */
+                        &callbacks));
+                } else {
+                    // Fall back to use the classic plan cache. Remove this branch after
+                    // "gFeatureFlagSbePlanCache" is removed.
+                    cacheClassicPlan();
+                }
+            } else {
+                static_assert(std::is_same_v<PlanStageType, PlanStage*>);
+                cacheClassicPlan();
+            }
         }
     }
 }
diff --git a/src/mongo/db/exec/sbe/values/slot.cpp b/src/mongo/db/exec/sbe/values/slot.cpp
index e0abb16dbec..e17ebccae70 100644
--- a/src/mongo/db/exec/sbe/values/slot.cpp
+++ b/src/mongo/db/exec/sbe/values/slot.cpp
@@ -466,6 +466,18 @@ int getApproximateSize(TypeTags tag, Value val) {
             // including the 'length' field itself.
             result += ConstDataView(getRawPointerView(val)).read<LittleEndian<uint32_t>>();
             break;
+        // These types are currently unsupported. Temporarily break in these cases because
+        // calculating the estimated size of a cached SBE plan requires a size value.
+        // TODO: SERVER-60273 Calculate the estimated size for missing types.
+        case TypeTags::LocalLambda:
+        case TypeTags::pcreRegex:
+        case TypeTags::timeZoneDB:
+        case TypeTags::jsFunction:
+        case TypeTags::shardFilterer:
+        case TypeTags::collator:
+        case TypeTags::ftsMatcher:
+        case TypeTags::sortSpec:
+            break;
         default:
             MONGO_UNREACHABLE;
     }