diff options
author | Ruoxin Xu <ruoxin.xu@mongodb.com> | 2021-09-07 11:17:00 +0100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-10-01 15:47:47 +0000 |
commit | b72d07a8b0f1f5d5935ae039196e9cd6531b3406 (patch) | |
tree | ed6a4d37fc0cf3a294b9a0cab3060459c4819972 | |
parent | fc05532015895c8907437ea0c06fe83ab6c6f1dc (diff) | |
download | mongo-b72d07a8b0f1f5d5935ae039196e9cd6531b3406.tar.gz |
SERVER-59681 Start storing SBE plans in the new plan cache
25 files changed, 192 insertions, 2 deletions
diff --git a/jstests/core/collation_plan_cache.js b/jstests/core/collation_plan_cache.js index 50e6921c91b..e9c25a36280 100644 --- a/jstests/core/collation_plan_cache.js +++ b/jstests/core/collation_plan_cache.js @@ -11,6 +11,12 @@ // ] (function() { 'use strict'; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} var coll = db.collation_plan_cache; coll.drop(); diff --git a/jstests/core/index_filter_commands.js b/jstests/core/index_filter_commands.js index 25ff7fc39ac..0a27a1ae5e1 100644 --- a/jstests/core/index_filter_commands.js +++ b/jstests/core/index_filter_commands.js @@ -39,6 +39,10 @@ load("jstests/libs/fixture_helpers.js"); // For 'FixtureHelpers'. load("jstests/libs/sbe_explain_helpers.js"); // For 'assertIdHackPlan()'. load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} const coll = db.jstests_index_filter_commands; coll.drop(); diff --git a/jstests/core/introspect_hidden_index_plan_cache_entries.js b/jstests/core/introspect_hidden_index_plan_cache_entries.js index 91645f46b5b..1af13d32d41 100644 --- a/jstests/core/introspect_hidden_index_plan_cache_entries.js +++ b/jstests/core/introspect_hidden_index_plan_cache_entries.js @@ -13,6 +13,12 @@ (function() { 'use strict'; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} const collName = 'introspect_hidden_index_plan_cache_entries'; const collNotAffectedName = 'introspect_hidden_index_plan_cache_entries_unaffected'; diff --git a/jstests/core/plan_cache_clear.js b/jstests/core/plan_cache_clear.js index f88e170bec6..02eed91081c 100644 --- a/jstests/core/plan_cache_clear.js +++ b/jstests/core/plan_cache_clear.js @@ -14,6 +14,14 @@ // ] (function() { +'use strict'; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} + const coll = db.jstests_plan_cache_clear; coll.drop(); diff --git a/jstests/core/plan_cache_list_plans.js b/jstests/core/plan_cache_list_plans.js index 13c28cfb1a8..c098acc6ee5 100644 --- a/jstests/core/plan_cache_list_plans.js +++ b/jstests/core/plan_cache_list_plans.js @@ -16,6 +16,13 @@ (function() { "use strict"; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} + let coll = db.jstests_plan_cache_list_plans; coll.drop(); diff --git a/jstests/core/plan_cache_list_shapes.js b/jstests/core/plan_cache_list_shapes.js index 66997c95ded..8e6a86b1494 100644 --- a/jstests/core/plan_cache_list_shapes.js +++ b/jstests/core/plan_cache_list_shapes.js @@ -13,6 +13,14 @@ // assumes_unsharded_collection, // ] (function() { +'use strict'; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} + const coll = db.jstests_plan_cache_list_shapes; coll.drop(); diff --git a/jstests/core/plan_cache_sbe.js b/jstests/core/plan_cache_sbe.js index 6069dae2e25..2cd26803858 100644 --- a/jstests/core/plan_cache_sbe.js +++ b/jstests/core/plan_cache_sbe.js @@ -19,6 +19,11 @@ load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} + const coll = db.plan_cache_sbe; coll.drop(); diff --git a/jstests/core/plan_cache_shell_helpers.js b/jstests/core/plan_cache_shell_helpers.js index aba0d20047e..5bd809988cb 100644 --- a/jstests/core/plan_cache_shell_helpers.js +++ b/jstests/core/plan_cache_shell_helpers.js @@ -12,6 +12,14 @@ // does_not_support_stepdowns, // ] (function() { +'use strict'; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} + var coll = db.jstests_plan_cache_shell_helpers; coll.drop(); diff --git a/jstests/core/plan_cache_stats_shard_and_host.js b/jstests/core/plan_cache_stats_shard_and_host.js index e81dcd60692..a6bc62e13db 100644 --- a/jstests/core/plan_cache_stats_shard_and_host.js +++ b/jstests/core/plan_cache_stats_shard_and_host.js @@ -11,6 +11,12 @@ "use strict"; load("jstests/libs/fixture_helpers.js"); // For 'FixtureHelpers'. +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} const coll = db.plan_cache_stats_shard_and_host; coll.drop(); diff --git a/jstests/core/profile_find.js b/jstests/core/profile_find.js index 05d9ee3a757..d69a2ce5754 100644 --- a/jstests/core/profile_find.js +++ b/jstests/core/profile_find.js @@ -10,6 +10,12 @@ // For 'getLatestProfilerEntry()'. load("jstests/libs/profiler.js"); +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} var testDB = db.getSiblingDB("profile_find"); assert.commandWorked(testDB.dropDatabase()); diff --git a/jstests/core/profile_query_hash.js b/jstests/core/profile_query_hash.js index 51b00d55996..6b88cbf13b8 100644 --- a/jstests/core/profile_query_hash.js +++ b/jstests/core/profile_query_hash.js @@ -11,6 +11,12 @@ // For getLatestProfilerEntry load("jstests/libs/profiler.js"); +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} const testDB = db.getSiblingDB("query_hash"); assert.commandWorked(testDB.dropDatabase()); diff --git a/jstests/core/wildcard_index_cached_plans.js b/jstests/core/wildcard_index_cached_plans.js index 6e98199dbe0..50d97aef1ad 100644 --- a/jstests/core/wildcard_index_cached_plans.js +++ b/jstests/core/wildcard_index_cached_plans.js @@ -22,6 +22,11 @@ load("jstests/libs/collection_drop_recreate.js"); // For assert[Drop|Create]Col load('jstests/libs/fixture_helpers.js'); // For getPrimaryForNodeHostingDatabase and isMongos. load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} + const coll = db.wildcard_cached_plans; coll.drop(); diff --git a/jstests/libs/sbe_util.js b/jstests/libs/sbe_util.js index a47aa3a1130..bb9bdd0d1b7 100644 --- a/jstests/libs/sbe_util.js +++ b/jstests/libs/sbe_util.js @@ -8,8 +8,9 @@ load("jstests/libs/fixture_helpers.js"); // For 'isMongos' /** * Returns whether or not SBE is enabled for the given connection. Assumes that for repl sets and * sharded clusters, SBE is either enabled on each node, or disabled on each node. + * If 'featureFlags' is non-empty, checks if SBE and all the feature flags are enabled. */ -function checkSBEEnabled(theDB) { +function checkSBEEnabled(theDB, featureFlags = []) { let checkResult = true; assert.soon(() => { @@ -32,6 +33,12 @@ function checkSBEEnabled(theDB) { continue; } + featureFlags.forEach(function(featureFlag) { + const featureFlagParam = conn.adminCommand({getParameter: 1, [featureFlag]: 1}); + checkResult = checkResult && featureFlagParam.hasOwnProperty(featureFlag) && + featureFlagParam[featureFlag]["value"]; + }); + const getParam = conn.adminCommand({getParameter: 1, internalQueryForceClassicEngine: 1}); if (getParam.hasOwnProperty("internalQueryForceClassicEngine") && diff --git a/jstests/noPassthrough/log_and_profile_query_hash.js b/jstests/noPassthrough/log_and_profile_query_hash.js index e4344eee337..5da528a8fef 100644 --- a/jstests/noPassthrough/log_and_profile_query_hash.js +++ b/jstests/noPassthrough/log_and_profile_query_hash.js @@ -7,6 +7,7 @@ // For getLatestProfilerEntry(). load("jstests/libs/profiler.js"); load("jstests/libs/logv2_helpers.js"); +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. // Prevent the mongo shell from gossiping its cluster time, since this will increase the amount // of data logged for each op. For some of the testcases below, including the cluster time would @@ -19,6 +20,12 @@ assert.neq(null, conn, "mongod was unable to start up"); const testDB = conn.getDB("jstests_query_shape_hash"); const coll = testDB.test; +if (checkSBEEnabled(testDB, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + MongoRunner.stopMongod(conn); + return; +} + const profileEntryFilter = { op: "query" }; diff --git a/jstests/noPassthrough/plan_cache_index_create.js b/jstests/noPassthrough/plan_cache_index_create.js index 7e9bb8efd89..3ce1731b847 100644 --- a/jstests/noPassthrough/plan_cache_index_create.js +++ b/jstests/noPassthrough/plan_cache_index_create.js @@ -9,6 +9,7 @@ "use strict"; load('jstests/libs/analyze_plan.js'); // For getCachedPlan(). +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. const dbName = "test"; const collName = "coll"; @@ -177,6 +178,12 @@ rst.initiate(); const primaryDB = rst.getPrimary().getDB(dbName); const secondaryDB = rst.getSecondary().getDB(dbName); +if (checkSBEEnabled(primaryDB, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + rst.stopSet(); + return; +} + runTest({rst: rst, readDB: primaryDB, writeDB: primaryDB}); runTest({rst: rst, readDB: secondaryDB, writeDB: primaryDB}); diff --git a/jstests/noPassthrough/plan_cache_list_failed_plans.js b/jstests/noPassthrough/plan_cache_list_failed_plans.js index 2fa5b641b2f..e1fb5161c71 100644 --- a/jstests/noPassthrough/plan_cache_list_failed_plans.js +++ b/jstests/noPassthrough/plan_cache_list_failed_plans.js @@ -10,6 +10,12 @@ const testDB = conn.getDB("jstests_plan_cache_list_failed_plans"); const coll = testDB.test; const isSBEEnabled = checkSBEEnabled(testDB); +if (checkSBEEnabled(testDB, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + MongoRunner.stopMongod(conn); + return; +} + coll.drop(); // Setup the database such that it will generate a failing plan and a succeeding plan. diff --git a/jstests/noPassthrough/plan_cache_memory_debug_info.js b/jstests/noPassthrough/plan_cache_memory_debug_info.js index a27b639665b..60406402158 100644 --- a/jstests/noPassthrough/plan_cache_memory_debug_info.js +++ b/jstests/noPassthrough/plan_cache_memory_debug_info.js @@ -4,6 +4,7 @@ */ (function() { "use strict"; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. /** * Creates two indexes for the given collection. In order for plans to be cached, there need to be @@ -75,6 +76,12 @@ const conn = MongoRunner.runMongod({}); assert.neq(conn, null, "mongod failed to start"); const db = conn.getDB("test"); const coll = db.plan_cache_memory_debug_info; + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + MongoRunner.stopMongod(conn); + return; +} coll.drop(); createIndexesForColl(coll); diff --git a/jstests/noPassthrough/plan_cache_metrics.js b/jstests/noPassthrough/plan_cache_metrics.js index 0ef39bfd753..21a114bf12c 100644 --- a/jstests/noPassthrough/plan_cache_metrics.js +++ b/jstests/noPassthrough/plan_cache_metrics.js @@ -4,6 +4,8 @@ */ (function() { "use strict"; +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + const conn = MongoRunner.runMongod({}); const db = conn.getDB('test'); const coll1 = db.query_metrics1; @@ -11,6 +13,12 @@ const coll2 = db.query_metrics2; coll1.drop(); coll2.drop(); +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + MongoRunner.stopMongod(conn); + return; +} + const queryObj = { a: {$gte: 99}, b: -1 diff --git a/jstests/noPassthrough/plan_cache_replan_sort.js b/jstests/noPassthrough/plan_cache_replan_sort.js index 04eacc37873..6683b502592 100644 --- a/jstests/noPassthrough/plan_cache_replan_sort.js +++ b/jstests/noPassthrough/plan_cache_replan_sort.js @@ -10,12 +10,19 @@ load("jstests/libs/analyze_plan.js"); load("jstests/libs/profiler.js"); +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. const conn = MongoRunner.runMongod(); const db = conn.getDB("test"); const coll = db.plan_cache_replan_sort; coll.drop(); +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + MongoRunner.stopMongod(conn); + return; +} + // Ensure a plan with a sort stage gets cached. assert.commandWorked(coll.createIndex({x: 1})); assert.commandWorked(coll.createIndex({y: 1})); diff --git a/jstests/noPassthrough/plan_cache_stats_agg_source.js b/jstests/noPassthrough/plan_cache_stats_agg_source.js index c34eb879b49..cedec94cd3f 100644 --- a/jstests/noPassthrough/plan_cache_stats_agg_source.js +++ b/jstests/noPassthrough/plan_cache_stats_agg_source.js @@ -13,6 +13,12 @@ assert.neq(null, conn, "mongod failed to start up"); const testDb = conn.getDB("test"); const coll = testDb.plan_cache_stats_agg_source; +if (checkSBEEnabled(testDb, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + MongoRunner.stopMongod(conn); + return; +} + // Note that the "getParameter" command is expected to fail in versions of mongod that do not yet // include the slot-based execution engine. When that happens, however, 'isSBEEnabled' still // correctly evaluates to false. diff --git a/jstests/noPassthroughWithMongod/ne_array_indexability.js b/jstests/noPassthroughWithMongod/ne_array_indexability.js index 606956c5753..8c88f09db8e 100644 --- a/jstests/noPassthroughWithMongod/ne_array_indexability.js +++ b/jstests/noPassthroughWithMongod/ne_array_indexability.js @@ -2,9 +2,16 @@ * Test that $ne: [] queries are cached correctly. See SERVER-39764. */ (function() { +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + const coll = db.ne_array_indexability; coll.drop(); +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} + coll.createIndex({"obj": 1}); coll.createIndex({"obj": 1, "abc": 1}); diff --git a/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js b/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js index 8a39fda391e..aec262b01e2 100644 --- a/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js +++ b/jstests/noPassthroughWithMongod/plan_cache_not_in_regex.js @@ -6,6 +6,12 @@ "use strict"; load('jstests/libs/analyze_plan.js'); // For isCollScan. +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} const coll = db.plan_cache_not_in_regex; coll.drop(); diff --git a/jstests/noPassthroughWithMongod/plan_cache_replanning.js b/jstests/noPassthroughWithMongod/plan_cache_replanning.js index be6a1c48f7b..c2043833c9d 100644 --- a/jstests/noPassthroughWithMongod/plan_cache_replanning.js +++ b/jstests/noPassthroughWithMongod/plan_cache_replanning.js @@ -8,6 +8,12 @@ load('jstests/libs/analyze_plan.js'); // For getPlanStage(). load("jstests/libs/collection_drop_recreate.js"); // For assert[Drop|Create]Collection. +load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. + +if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) { + jsTest.log("Skipping test because SBE and SBE plan cache are both enabled."); + return; +} let coll = assertDropAndRecreateCollection(db, "plan_cache_replanning"); diff --git a/src/mongo/db/exec/plan_cache_util.h b/src/mongo/db/exec/plan_cache_util.h index 5ef7cd8fd45..a03ac6a9eb2 100644 --- a/src/mongo/db/exec/plan_cache_util.h +++ b/src/mongo/db/exec/plan_cache_util.h @@ -34,6 +34,7 @@ #include "mongo/db/query/collection_query_info.h" #include "mongo/db/query/plan_cache_key_factory.h" #include "mongo/db/query/plan_explainer_factory.h" +#include "mongo/db/query/sbe_plan_cache.h" #include "mongo/db/query/sbe_plan_ranker.h" namespace mongo { @@ -181,7 +182,7 @@ void updatePlanCache( } } - if (validSolutions) { + auto cacheClassicPlan = [&]() { // The caller of this constructor is responsible for ensuring that the QuerySolution // has valid cacheData. If there's no data to cache you shouldn't be trying to // construct a PlanCacheEntry. @@ -202,6 +203,34 @@ void updatePlanCache( opCtx->getServiceContext()->getPreciseClockSource()->now(), boost::none, /* worksGrowthCoefficient */ &callbacks)); + }; + + if (validSolutions) { + if constexpr (std::is_same_v<PlanStageType, std::unique_ptr<sbe::PlanStage>>) { + if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) { + // Clone the winning SBE plan and its auxiliary data. + auto cachedPlan = std::make_unique<sbe::CachedSbePlan>( + candidates[winnerIdx].root->clone(), candidates[winnerIdx].data); + + PlanCacheLoggingCallbacks<sbe::PlanCacheKey, sbe::CachedSbePlan> callbacks{ + query}; + uassertStatusOK(sbe::getPlanCache(opCtx).set( + plan_cache_key_factory::make<sbe::PlanCacheKey>(query, collection), + std::move(cachedPlan), + solutions, + std::move(ranking), + opCtx->getServiceContext()->getPreciseClockSource()->now(), + boost::none, /* worksGrowthCoefficient */ + &callbacks)); + } else { + // Fall back to use the classic plan cache. Remove this branch after + // "gFeatureFlagSbePlanCache" is removed. + cacheClassicPlan(); + } + } else { + static_assert(std::is_same_v<PlanStageType, PlanStage*>); + cacheClassicPlan(); + } } } } diff --git a/src/mongo/db/exec/sbe/values/slot.cpp b/src/mongo/db/exec/sbe/values/slot.cpp index e0abb16dbec..e17ebccae70 100644 --- a/src/mongo/db/exec/sbe/values/slot.cpp +++ b/src/mongo/db/exec/sbe/values/slot.cpp @@ -466,6 +466,18 @@ int getApproximateSize(TypeTags tag, Value val) { // including the 'length' field itself. result += ConstDataView(getRawPointerView(val)).read<LittleEndian<uint32_t>>(); break; + // These types are currently unsupported. Temporarily break in these cases because + // calculating the estimated size of a cached SBE plan requires a size value. + // TODO: SERVER-60273 Calculate the estimated size for missing types. + case TypeTags::LocalLambda: + case TypeTags::pcreRegex: + case TypeTags::timeZoneDB: + case TypeTags::jsFunction: + case TypeTags::shardFilterer: + case TypeTags::collator: + case TypeTags::ftsMatcher: + case TypeTags::sortSpec: + break; default: MONGO_UNREACHABLE; } |