diff options
author | Ruoxin Xu <ruoxin.xu@mongodb.com> | 2022-01-17 18:13:45 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-01-17 18:39:52 +0000 |
commit | a29b0a6f075bb05a1a87927508edd31656a6a15c (patch) | |
tree | 6111105c737e775d74841f7775a6abee9f483db4 /src | |
parent | 59d341f677f355939c6f4e8e9934ea1de700c1f7 (diff) | |
download | mongo-a29b0a6f075bb05a1a87927508edd31656a6a15c.tar.gz |
SERVER-59682 Recover SBE plans from the new plan cache
Diffstat (limited to 'src')
30 files changed, 943 insertions, 468 deletions
diff --git a/src/mongo/db/commands/index_filter_commands_test.cpp b/src/mongo/db/commands/index_filter_commands_test.cpp index 932c2280f92..7a03ec7dcda 100644 --- a/src/mongo/db/commands/index_filter_commands_test.cpp +++ b/src/mongo/db/commands/index_filter_commands_test.cpp @@ -36,6 +36,7 @@ #include <memory> #include "mongo/db/catalog/collection_mock.h" +#include "mongo/db/exec/plan_cache_util.h" #include "mongo/db/json.h" #include "mongo/db/operation_context_noop.h" #include "mongo/db/query/collation/collator_interface_mock.h" @@ -148,11 +149,15 @@ void addQueryShapeToPlanCache(OperationContext* opCtx, auto cacheData = std::make_unique<SolutionCacheData>(); cacheData->tree = std::make_unique<PlanCacheIndexTree>(); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*cq}; + auto decision = createDecision(1U); + auto decisionPtr = decision.get(); + PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData, plan_cache_debug_info::DebugInfo> + callbacks{*cq}; ASSERT_OK(planCache->set(makeKey(*cq), std::move(cacheData), - createDecision(1U), + *decisionPtr, opCtx->getServiceContext()->getPreciseClockSource()->now(), + plan_cache_util::buildDebugInfo(*cq, std::move(decision)), boost::none, /* worksGrowthCoefficient */ &callbacks)); } diff --git a/src/mongo/db/exec/plan_cache_util.cpp b/src/mongo/db/exec/plan_cache_util.cpp index ef72a9d553b..0a4aab74886 100644 --- a/src/mongo/db/exec/plan_cache_util.cpp +++ b/src/mongo/db/exec/plan_cache_util.cpp @@ -32,9 +32,11 @@ #include "mongo/platform/basic.h" #include "mongo/db/exec/plan_cache_util.h" + #include "mongo/logv2/log.h" -namespace mongo::plan_cache_util { +namespace mongo { +namespace plan_cache_util { namespace log_detail { void logTieForBest(std::string&& query, double winnerScore, @@ -67,4 +69,96 @@ void logNotCachingNoData(std::string&& solution) { "solutions"_attr = redact(solution)); } } // namespace log_detail -} // namespace mongo::plan_cache_util + +plan_cache_debug_info::DebugInfo buildDebugInfo( + const CanonicalQuery& query, std::unique_ptr<const plan_ranker::PlanRankingDecision> decision) { + // Strip projections on $-prefixed fields, as these are added by internal callers of the + // system and are not considered part of the user projection. + const FindCommandRequest& findCommand = query.getFindCommandRequest(); + BSONObjBuilder projBuilder; + for (auto elem : findCommand.getProjection()) { + if (elem.fieldName()[0] == '$') { + continue; + } + projBuilder.append(elem); + } + + plan_cache_debug_info::CreatedFromQuery createdFromQuery = + plan_cache_debug_info::CreatedFromQuery{ + findCommand.getFilter(), + findCommand.getSort(), + projBuilder.obj(), + query.getCollator() ? query.getCollator()->getSpec().toBSON() : BSONObj()}; + + return {std::move(createdFromQuery), std::move(decision)}; +} + +plan_cache_debug_info::DebugInfoSBE buildDebugInfo(const QuerySolution* solution) { + plan_cache_debug_info::DebugInfoSBE debugInfo; + + if (!solution || !solution->root()) + return debugInfo; + + std::queue<const QuerySolutionNode*> queue; + queue.push(solution->root()); + + // Look through the QuerySolution to collect some static stat details. + while (!queue.empty()) { + auto node = queue.front(); + queue.pop(); + invariant(node); + + switch (node->getType()) { + case STAGE_COUNT_SCAN: { + auto csn = static_cast<const CountScanNode*>(node); + debugInfo.indexesUsed.push_back(csn->index.identifier.catalogName); + break; + } + case STAGE_DISTINCT_SCAN: { + auto dn = static_cast<const DistinctNode*>(node); + debugInfo.indexesUsed.push_back(dn->index.identifier.catalogName); + break; + } + case STAGE_GEO_NEAR_2D: { + auto geo2d = static_cast<const GeoNear2DNode*>(node); + debugInfo.indexesUsed.push_back(geo2d->index.identifier.catalogName); + break; + } + case STAGE_GEO_NEAR_2DSPHERE: { + auto geo2dsphere = static_cast<const GeoNear2DSphereNode*>(node); + debugInfo.indexesUsed.push_back(geo2dsphere->index.identifier.catalogName); + break; + } + case STAGE_IXSCAN: { + auto ixn = static_cast<const IndexScanNode*>(node); + debugInfo.indexesUsed.push_back(ixn->index.identifier.catalogName); + break; + } + case STAGE_TEXT_MATCH: { + auto tn = static_cast<const TextMatchNode*>(node); + debugInfo.indexesUsed.push_back(tn->index.identifier.catalogName); + break; + } + case STAGE_COLLSCAN: { + debugInfo.collectionScans++; + auto csn = static_cast<const CollectionScanNode*>(node); + if (!csn->tailable) { + debugInfo.collectionScansNonTailable++; + } + break; + } + default: + break; + } + + for (auto&& child : node->children) { + queue.push(child); + } + } + + debugInfo.planSummary = solution->summaryString(); + + return debugInfo; +} +} // namespace plan_cache_util +} // namespace mongo diff --git a/src/mongo/db/exec/plan_cache_util.h b/src/mongo/db/exec/plan_cache_util.h index 47ae121609b..9b8851fac22 100644 --- a/src/mongo/db/exec/plan_cache_util.h +++ b/src/mongo/db/exec/plan_cache_util.h @@ -32,8 +32,10 @@ #include "mongo/db/exec/plan_stats.h" #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/collection_query_info.h" +#include "mongo/db/query/plan_cache_debug_info.h" #include "mongo/db/query/plan_cache_key_factory.h" #include "mongo/db/query/plan_explainer_factory.h" +#include "mongo/db/query/query_solution.h" #include "mongo/db/query/sbe_plan_cache.h" #include "mongo/db/query/sbe_plan_ranker.h" @@ -71,6 +73,18 @@ void logNotCachingZeroResults(std::string&& query, double score, std::string win void logNotCachingNoData(std::string&& solution); } // namespace log_detail +/* + * Builds "DebugInfo" for storing in the classic plan cache. + */ +plan_cache_debug_info::DebugInfo buildDebugInfo( + const CanonicalQuery& query, std::unique_ptr<const plan_ranker::PlanRankingDecision> decision); + +/* + * Builds "DebugInfoSBE" for storing in the SBE plan cache. Pre-computes necessary debugging + * information to build "PlanExplainerSBE" when recoverying the cached SBE plan from the cache. + */ +plan_cache_debug_info::DebugInfoSBE buildDebugInfo(const QuerySolution* solution); + /** * Caches the best candidate plan, chosen from the given 'candidates' based on the 'ranking' * decision, if the 'query' is of a type that can be cached. Otherwise, does nothing. @@ -165,14 +179,19 @@ void updatePlanCache( // Store the choice we just made in the cache, if the query is of a type that is safe to // cache. if (shouldCacheQuery(query) && canCache) { + auto rankingDecision = ranking.get(); auto cacheClassicPlan = [&]() { - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{query}; + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + plan_cache_debug_info::DebugInfo> + callbacks{query}; uassertStatusOK(CollectionQueryInfo::get(collection) .getPlanCache() ->set(plan_cache_key_factory::make<PlanCacheKey>(query, collection), winningPlan.solution->cacheData->clone(), - std::move(ranking), + *rankingDecision, opCtx->getServiceContext()->getPreciseClockSource()->now(), + buildDebugInfo(query, std::move(ranking)), boost::none, /* worksGrowthCoefficient */ &callbacks)); }; @@ -184,13 +203,16 @@ void updatePlanCache( auto cachedPlan = std::make_unique<sbe::CachedSbePlan>( winningPlan.root->clone(), winningPlan.data); - PlanCacheLoggingCallbacks<sbe::PlanCacheKey, sbe::CachedSbePlan> callbacks{ - query}; + PlanCacheLoggingCallbacks<sbe::PlanCacheKey, + sbe::CachedSbePlan, + plan_cache_debug_info::DebugInfoSBE> + callbacks{query}; uassertStatusOK(sbe::getPlanCache(opCtx).set( plan_cache_key_factory::make<sbe::PlanCacheKey>(query, collection), std::move(cachedPlan), - std::move(ranking), + *rankingDecision, opCtx->getServiceContext()->getPreciseClockSource()->now(), + buildDebugInfo(winningPlan.solution.get()), boost::none, /* worksGrowthCoefficient */ &callbacks)); } else { diff --git a/src/mongo/db/exec/sbe/sbe_hash_agg_test.cpp b/src/mongo/db/exec/sbe/sbe_hash_agg_test.cpp index 7037bd1e86d..659c249b18d 100644 --- a/src/mongo/db/exec/sbe/sbe_hash_agg_test.cpp +++ b/src/mongo/db/exec/sbe/sbe_hash_agg_test.cpp @@ -64,7 +64,7 @@ void HashAggStageTest::performHashAggWithSpillChecking( value::ValueGuard expectedGuard{expectedTag, expectedVal}; auto collatorSlot = generateSlotId(); - auto shouldUseCollator = optionalCollator.get() != nullptr; + auto shouldUseCollator = optionalCollator != nullptr; auto makeStageFn = [this, collatorSlot, shouldUseCollator, shouldSpill]( value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { @@ -92,7 +92,7 @@ void HashAggStageTest::performHashAggWithSpillChecking( if (shouldUseCollator) { ctx->pushCorrelated(collatorSlot, &collatorAccessor); collatorAccessor.reset(value::TypeTags::collator, - value::bitcastFrom<CollatorInterface*>(optionalCollator.get())); + value::bitcastFrom<CollatorInterface*>(optionalCollator.release())); } // Generate a mock scan from 'input' with a single output slot. @@ -155,8 +155,8 @@ TEST_F(HashAggStageTest, HashAggMinMaxTest) { auto makeStageFn = [this, &collator](value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { - auto collExpr = makeE<EConstant>(value::TypeTags::collator, - value::bitcastFrom<CollatorInterface*>(collator.get())); + auto collExpr = makeE<EConstant>( + value::TypeTags::collator, value::bitcastFrom<CollatorInterface*>(collator.release())); // Build a HashAggStage that exercises the collMin() and collMax() aggregate functions. auto minSlot = generateSlotId(); @@ -222,8 +222,8 @@ TEST_F(HashAggStageTest, HashAggAddToSetTest) { auto makeStageFn = [this, &collator](value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { - auto collExpr = makeE<EConstant>(value::TypeTags::collator, - value::bitcastFrom<CollatorInterface*>(collator.get())); + auto collExpr = makeE<EConstant>( + value::TypeTags::collator, value::bitcastFrom<CollatorInterface*>(collator.release())); // Build a HashAggStage that exercises the collAddToSet() aggregate function. auto hashAggSlot = generateSlotId(); diff --git a/src/mongo/db/exec/sbe/sbe_hash_join_test.cpp b/src/mongo/db/exec/sbe/sbe_hash_join_test.cpp index 3eb25c4ebf4..4bdcedb31a0 100644 --- a/src/mongo/db/exec/sbe/sbe_hash_join_test.cpp +++ b/src/mongo/db/exec/sbe/sbe_hash_join_test.cpp @@ -92,7 +92,7 @@ TEST_F(HashJoinStageTest, HashJoinCollationTest) { value::OwnedValueAccessor collatorAccessor; ctx->pushCorrelated(collatorSlot, &collatorAccessor); collatorAccessor.reset(value::TypeTags::collator, - value::bitcastFrom<CollatorInterface*>(collator.get())); + value::bitcastFrom<CollatorInterface*>(collator.release())); // Two separate virtual scans are needed since HashJoinStage needs two child stages. outerGuard.reset(); diff --git a/src/mongo/db/exec/sbe/stages/stages.h b/src/mongo/db/exec/sbe/stages/stages.h index 555dc05aaa5..9afd8f7de64 100644 --- a/src/mongo/db/exec/sbe/stages/stages.h +++ b/src/mongo/db/exec/sbe/stages/stages.h @@ -360,6 +360,7 @@ private: * Maintains an internal state to maintain the interrupt check period. Also responsible for * triggering yields if this object has been configured with a yield policy. */ +template <typename T> class CanInterrupt { public: /** @@ -389,8 +390,17 @@ public: } } + void attachNewYieldPolicy(PlanYieldPolicy* yieldPolicy) { + auto stage = static_cast<T*>(this); + for (auto&& child : stage->_children) { + child->attachNewYieldPolicy(yieldPolicy); + } + + _yieldPolicy = yieldPolicy; + } + protected: - PlanYieldPolicy* const _yieldPolicy{nullptr}; + PlanYieldPolicy* _yieldPolicy{nullptr}; private: static const int kInterruptCheckPeriod = 128; @@ -403,7 +413,7 @@ private: class PlanStage : public CanSwitchOperationContext<PlanStage>, public CanChangeState<PlanStage>, public CanTrackStats<PlanStage>, - public CanInterrupt { + public CanInterrupt<PlanStage> { public: using Vector = absl::InlinedVector<std::unique_ptr<PlanStage>, 2>; @@ -470,6 +480,7 @@ public: friend class CanSwitchOperationContext<PlanStage>; friend class CanChangeState<PlanStage>; friend class CanTrackStats<PlanStage>; + friend class CanInterrupt<PlanStage>; protected: // Derived classes can optionally override these methods. diff --git a/src/mongo/db/exec/sbe/values/value.cpp b/src/mongo/db/exec/sbe/values/value.cpp index 0e2e19d08a5..8c7de0f1b8e 100644 --- a/src/mongo/db/exec/sbe/values/value.cpp +++ b/src/mongo/db/exec/sbe/values/value.cpp @@ -243,6 +243,11 @@ std::pair<TypeTags, Value> makeCopySortSpec(const SortSpec& ss) { return {TypeTags::sortSpec, ssCopy}; } +std::pair<TypeTags, Value> makeCopyCollator(const CollatorInterface& collator) { + auto collatorCopy = bitcastFrom<CollatorInterface*>(collator.clone().release()); + return {TypeTags::collator, collatorCopy}; +} + std::pair<TypeTags, Value> makeNewRecordId(int64_t rid) { auto val = bitcastFrom<RecordId*>(new RecordId(rid)); return {TypeTags::RecordId, val}; @@ -311,6 +316,9 @@ void releaseValue(TypeTags tag, Value val) noexcept { case TypeTags::sortSpec: delete getSortSpecView(val); break; + case TypeTags::collator: + delete getCollatorView(val); + break; default: break; } diff --git a/src/mongo/db/exec/sbe/values/value.h b/src/mongo/db/exec/sbe/values/value.h index b5947391347..6a7cfe810e8 100644 --- a/src/mongo/db/exec/sbe/values/value.h +++ b/src/mongo/db/exec/sbe/values/value.h @@ -1359,6 +1359,8 @@ std::pair<TypeTags, Value> makeCopyFtsMatcher(const fts::FTSMatcher&); std::pair<TypeTags, Value> makeCopySortSpec(const SortSpec&); +std::pair<TypeTags, Value> makeCopyCollator(const CollatorInterface& collator); + /** * Releases memory allocated for the value. If the value does not have any memory allocated for it, * does nothing. @@ -1434,6 +1436,8 @@ inline std::pair<TypeTags, Value> copyValue(TypeTags tag, Value val) { return makeCopyFtsMatcher(*getFtsMatcherView(val)); case TypeTags::sortSpec: return makeCopySortSpec(*getSortSpecView(val)); + case TypeTags::collator: + return makeCopyCollator(*getCollatorView(val)); default: break; } diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp index bb9fe54d46b..3067074f008 100644 --- a/src/mongo/db/pipeline/pipeline_d.cpp +++ b/src/mongo/db/pipeline/pipeline_d.cpp @@ -181,6 +181,22 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> attemptToGetExe if (aggRequest) { findCommand->setHint(aggRequest->getHint().value_or(BSONObj()).getOwned()); isExplain = static_cast<bool>(aggRequest->getExplain()); + // TODO SERVER-62100: No need to populate the "let" object to FindCommand for encoding. + if (auto let = aggRequest->getLet()) { + findCommand->setLet(let); + } + } + + // TODO SERVER-62100: No need to populate the "let" object to FindCommand for encoding. + if (!findCommand->getLet() && expCtx->variablesParseState.hasDefinedVariables()) { + auto varIds = expCtx->variablesParseState.getDefinedVariableIDs(); + for (auto&& id : varIds) { + // Only serialize "let" if there is user-defined "let" variable. + if (expCtx->variables.hasValue(id)) { + findCommand->setLet(expCtx->variablesParseState.serialize(expCtx->variables)); + break; + } + } } // The collation on the ExpressionContext has been resolved to either the user-specified diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index cf1cfea8d5f..2d07cdf4f25 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -89,7 +89,6 @@ env.Library( source=[ "classic_plan_cache.cpp", "plan_cache_callbacks.cpp", - "plan_cache_debug_info.cpp", "plan_cache_invalidator.cpp", "sbe_plan_cache.cpp", ], diff --git a/src/mongo/db/query/canonical_query_encoder.cpp b/src/mongo/db/query/canonical_query_encoder.cpp index 3164242d830..51dbff5d00e 100644 --- a/src/mongo/db/query/canonical_query_encoder.cpp +++ b/src/mongo/db/query/canonical_query_encoder.cpp @@ -645,22 +645,27 @@ std::string encodeSBE(const CanonicalQuery& cq) { const auto& filter = cq.getQueryObj(); const auto& proj = cq.getFindCommandRequest().getProjection(); const auto& sort = cq.getFindCommandRequest().getSort(); + const auto& let = cq.getFindCommandRequest().getLet(); StringBuilder strBuilder; encodeKeyForSort(sort, &strBuilder); encodeCollation(cq.getCollator(), &strBuilder); - auto sortAndCollation = strBuilder.stringData(); + auto strBuilderEncoded = strBuilder.stringData(); // A constant for reserving buffer size. It should be large enough to reserve the space required // to encode various properties from the FindCommandRequest and query knobs. const int kBufferSizeConstant = 200; - size_t bufSize = - filter.objsize() + proj.objsize() + sortAndCollation.size() + kBufferSizeConstant; + size_t bufSize = filter.objsize() + proj.objsize() + strBuilderEncoded.size() + + kBufferSizeConstant + (let ? let->objsize() : 0); BufBuilder bufBuilder(bufSize); bufBuilder.appendBuf(filter.objdata(), filter.objsize()); bufBuilder.appendBuf(proj.objdata(), proj.objsize()); - bufBuilder.appendStr(sortAndCollation, false /* includeEndingNull */); + // TODO SERVER-62100: No need to encode the entire "let" object. + if (let) { + bufBuilder.appendBuf(let->objdata(), let->objsize()); + } + bufBuilder.appendStr(strBuilderEncoded, false /* includeEndingNull */); encodeFindCommandRequest(cq.getFindCommandRequest(), &bufBuilder); encodeQueryParameters(&bufBuilder); diff --git a/src/mongo/db/query/classic_plan_cache.h b/src/mongo/db/query/classic_plan_cache.h index d6a05895b5d..3b38abd891f 100644 --- a/src/mongo/db/query/classic_plan_cache.h +++ b/src/mongo/db/query/classic_plan_cache.h @@ -230,9 +230,9 @@ struct SolutionCacheData { bool indexFilterApplied; }; -using PlanCacheEntry = PlanCacheEntryBase<SolutionCacheData>; +using PlanCacheEntry = PlanCacheEntryBase<SolutionCacheData, plan_cache_debug_info::DebugInfo>; -using CachedSolution = CachedPlanHolder<SolutionCacheData>; +using CachedSolution = CachedPlanHolder<SolutionCacheData, plan_cache_debug_info::DebugInfo>; struct BudgetEstimator { size_t operator()(const PlanCacheEntry&) { @@ -243,6 +243,7 @@ struct BudgetEstimator { using PlanCache = PlanCacheBase<PlanCacheKey, SolutionCacheData, BudgetEstimator, + plan_cache_debug_info::DebugInfo, PlanCachePartitioner, PlanCacheKeyHasher>; diff --git a/src/mongo/db/query/explain.cpp b/src/mongo/db/query/explain.cpp index 0469b9eab35..a3b66328725 100644 --- a/src/mongo/db/query/explain.cpp +++ b/src/mongo/db/query/explain.cpp @@ -94,15 +94,24 @@ void generatePlannerInfo(PlanExecutor* exec, if (collection && exec->getCanonicalQuery()) { const QuerySettings* querySettings = QuerySettingsDecoration::get(collection->getSharedDecorations()); - const auto planCacheKeyInfo = - plan_cache_key_factory::make<PlanCacheKey>(*exec->getCanonicalQuery(), collection); - planCacheKeyHash = planCacheKeyInfo.planCacheKeyHash(); - queryHash = planCacheKeyInfo.queryHash(); - - if (auto allowedIndicesFilter = - querySettings->getAllowedIndicesFilter(planCacheKeyInfo.getQueryShape())) { - // Found an index filter set on the query shape. - indexFilterSet = true; + if (exec->getCanonicalQuery()->isSbeCompatible() && + feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV() && + !exec->getCanonicalQuery()->getForceClassicEngine()) { + const auto planCacheKeyInfo = plan_cache_key_factory::make<sbe::PlanCacheKey>( + *exec->getCanonicalQuery(), collection); + planCacheKeyHash = planCacheKeyInfo.planCacheKeyHash(); + queryHash = planCacheKeyInfo.queryHash(); + // TODO SERVER-59695: Set the correct value of "indexFilterSet". + } else { + const auto planCacheKeyInfo = + plan_cache_key_factory::make<PlanCacheKey>(*exec->getCanonicalQuery(), collection); + planCacheKeyHash = planCacheKeyInfo.planCacheKeyHash(); + queryHash = planCacheKeyInfo.queryHash(); + if (auto allowedIndicesFilter = + querySettings->getAllowedIndicesFilter(planCacheKeyInfo.getQueryShape())) { + // Found an index filter set on the query shape. + indexFilterSet = true; + } } } plannerBob.append("indexFilterSet", indexFilterSet); diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index 3079e32f172..91f17d08a30 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -452,6 +452,17 @@ public: _solutions.push_back(std::move(solution)); } + void emplace(std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> debugInfo, + std::pair<std::unique_ptr<sbe::PlanStage>, stage_builder::PlanStageData> root) { + tassert(5968202, + "debugInfo should not be null and _debugInfo should", + debugInfo && !_debugInfo); + _debugInfo = std::move(debugInfo); + _roots.push_back(std::move(root)); + // Make sure we store an empty QuerySolution instead of a nullptr or nothing. + _solutions.push_back(std::make_unique<QuerySolution>()); + } + std::string getPlanSummary() const { // We can report plan summary only if this result contains a single solution. invariant(_roots.size() == 1); @@ -462,8 +473,11 @@ public: return explainer->getPlanSummary(); } - std::tuple<PlanStageVector, QuerySolutionVector> extractResultData() { - return std::make_tuple(std::move(_roots), std::move(_solutions)); + std::tuple<PlanStageVector, + QuerySolutionVector, + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE>> + extractResultData() { + return std::make_tuple(std::move(_roots), std::move(_solutions), std::move(_debugInfo)); } boost::optional<size_t> decisionWorks() const { @@ -487,6 +501,7 @@ private: PlanStageVector _roots; boost::optional<size_t> _decisionWorks; bool _needSubplanning{false}; + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> _debugInfo; }; /** @@ -498,7 +513,7 @@ private: * * We have a QuerySolutionNode tree (or multiple query solution trees), but must execute some * custom logic in order to build the final execution tree. */ -template <typename PlanStageType, typename ResultType> +template <typename KeyType, typename PlanStageType, typename ResultType> class PrepareExecutionHelper { public: PrepareExecutionHelper(OperationContext* opCtx, @@ -570,9 +585,8 @@ public: << " tailable cursor requested on non capped collection"); } + auto planCacheKey = plan_cache_key_factory::make<KeyType>(*_cq, _collection); // Fill in some opDebug information, unless it has already been filled by an outer pipeline. - const PlanCacheKey planCacheKey = - plan_cache_key_factory::make<PlanCacheKey>(*_cq, _collection); OpDebug& opDebug = CurOp::get(_opCtx)->debug(); if (!opDebug.queryHash) { opDebug.queryHash = planCacheKey.queryHash(); @@ -580,31 +594,9 @@ public: // Check that the query should be cached. if (shouldCacheQuery(*_cq)) { - // Fill in the 'planCacheKey' too if the query is actually being cached. - if (!opDebug.planCacheKey) { - opDebug.planCacheKey = planCacheKey.planCacheKeyHash(); - } - - // Try to look up a cached solution for the query. - if (auto cs = CollectionQueryInfo::get(_collection) - .getPlanCache() - ->getCacheEntryIfActive(planCacheKey)) { - // We have a CachedSolution. Have the planner turn it into a QuerySolution. - auto statusWithQs = QueryPlanner::planFromCache(*_cq, plannerParams, *cs); - - if (statusWithQs.isOK()) { - auto querySolution = std::move(statusWithQs.getValue()); - if ((plannerParams.options & QueryPlannerParams::IS_COUNT) && - turnIxscanIntoCount(querySolution.get())) { - LOGV2_DEBUG(20923, - 2, - "Using fast count", - "query"_attr = redact(_cq->toStringShort())); - } - - return buildCachedPlan( - std::move(querySolution), plannerParams, cs->decisionWorks); - } + auto result = buildCachedPlan(plannerParams, planCacheKey); + if (result) { + return {std::move(result)}; } } @@ -687,16 +679,11 @@ protected: QueryPlannerParams* plannerParams) = 0; /** - * Constructs a PlanStage tree from a cached plan and also: - * * Either modifies the constructed tree to run a trial period in order to evaluate the - * cost of a cached plan. If the cost is unexpectedly high, the plan cache entry is - * deactivated and we use multi-planning to select an entirely new winning plan. - * * Or stores additional information in the result object, in case runtime planning is - * implemented as a standalone component, rather than as part of the execution tree. + * Constructs a PlanStage tree from a cached plan (if exists in the plan cache). Returns + * nullptr if no cached plan found. */ - virtual std::unique_ptr<ResultType> buildCachedPlan(std::unique_ptr<QuerySolution> solution, - const QueryPlannerParams& plannerParams, - size_t decisionWorks) = 0; + virtual std::unique_ptr<ResultType> buildCachedPlan(const QueryPlannerParams& plannerParams, + const KeyType& planCacheKey) = 0; /** * Constructs a special PlanStage tree for rooted $or queries. Each clause of the $or is planned @@ -731,7 +718,9 @@ protected: * A helper class to prepare a classic PlanStage tree for execution. */ class ClassicPrepareExecutionHelper final - : public PrepareExecutionHelper<std::unique_ptr<PlanStage>, ClassicPrepareExecutionResult> { + : public PrepareExecutionHelper<PlanCacheKey, + std::unique_ptr<PlanStage>, + ClassicPrepareExecutionResult> { public: ClassicPrepareExecutionHelper(OperationContext* opCtx, const CollectionPtr& collection, @@ -812,25 +801,48 @@ protected: } std::unique_ptr<ClassicPrepareExecutionResult> buildCachedPlan( - std::unique_ptr<QuerySolution> solution, - const QueryPlannerParams& plannerParams, - size_t decisionWorks) final { - auto result = makeResult(); - auto&& root = buildExecutableTree(*solution); - - // Add a CachedPlanStage on top of the previous root. - // - // 'decisionWorks' is used to determine whether the existing cache entry should - // be evicted, and the query replanned. - result->emplace(std::make_unique<CachedPlanStage>(_cq->getExpCtxRaw(), - _collection, - _ws, - _cq, - plannerParams, - decisionWorks, - std::move(root)), - std::move(solution)); - return result; + const QueryPlannerParams& plannerParams, const PlanCacheKey& planCacheKey) final { + OpDebug& opDebug = CurOp::get(_opCtx)->debug(); + if (!opDebug.planCacheKey) { + opDebug.planCacheKey = planCacheKey.planCacheKeyHash(); + } + // Try to look up a cached solution for the query. + if (auto cs = CollectionQueryInfo::get(_collection) + .getPlanCache() + ->getCacheEntryIfActive(planCacheKey)) { + // We have a CachedSolution. Have the planner turn it into a QuerySolution. + auto statusWithQs = QueryPlanner::planFromCache(*_cq, plannerParams, *cs); + + if (statusWithQs.isOK()) { + auto querySolution = std::move(statusWithQs.getValue()); + if ((plannerParams.options & QueryPlannerParams::IS_COUNT) && + turnIxscanIntoCount(querySolution.get())) { + LOGV2_DEBUG(5968201, + 2, + "Using fast count", + "query"_attr = redact(_cq->toStringShort())); + } + + auto result = makeResult(); + auto&& root = buildExecutableTree(*querySolution); + + // Add a CachedPlanStage on top of the previous root. + // + // 'decisionWorks' is used to determine whether the existing cache entry should + // be evicted, and the query replanned. + result->emplace(std::make_unique<CachedPlanStage>(_cq->getExpCtxRaw(), + _collection, + _ws, + _cq, + plannerParams, + cs->decisionWorks, + std::move(root)), + std::move(querySolution)); + return result; + } + } + + return nullptr; } std::unique_ptr<ClassicPrepareExecutionResult> buildSubPlan( @@ -875,6 +887,7 @@ private: */ class SlotBasedPrepareExecutionHelper final : public PrepareExecutionHelper< + sbe::PlanCacheKey, std::pair<std::unique_ptr<sbe::PlanStage>, stage_builder::PlanStageData>, SlotBasedPrepareExecutionResult> { public: @@ -950,16 +963,87 @@ protected: } std::unique_ptr<SlotBasedPrepareExecutionResult> buildCachedPlan( - std::unique_ptr<QuerySolution> solution, - const QueryPlannerParams& plannerParams, - size_t decisionWorks) final { + const QueryPlannerParams& plannerParams, const sbe::PlanCacheKey& planCacheKey) final { + if (!feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) { + // If the feature flag is off we fall back to use the classic plan cache just as what we + // do in caching SBE plans. + return buildCachedPlanFromClassicCache(plannerParams); + } + + OpDebug& opDebug = CurOp::get(_opCtx)->debug(); + if (!opDebug.planCacheKey) { + opDebug.planCacheKey = planCacheKey.planCacheKeyHash(); + } + + auto&& planCache = sbe::getPlanCache(_opCtx); + auto cacheEntry = planCache.getCacheEntryIfActive(planCacheKey); + if (!cacheEntry) { + return nullptr; + } + + auto&& cachedPlan = std::move(cacheEntry->cachedPlan); + auto root = std::move(cachedPlan->root); + auto stageData = std::move(cachedPlan->planStageData); + + root->attachToOperationContext(_opCtx); + root->attachNewYieldPolicy(_yieldPolicy); + + auto expCtx = _cq->getExpCtxRaw(); + tassert(5968200, "No expression context", expCtx); + if (expCtx->explain || expCtx->mayDbProfile) { + root->markShouldCollectTimingInfo(); + } + + // Register this plan to yield according to the configured policy. + auto sbeYieldPolicy = dynamic_cast<PlanYieldPolicySBE*>(_yieldPolicy); + invariant(sbeYieldPolicy); + sbeYieldPolicy->registerPlan(root.get()); + auto result = makeResult(); - auto execTree = buildExecutableTree(*solution); - result->emplace(std::move(execTree), std::move(solution)); - result->setDecisionWorks(decisionWorks); + result->setDecisionWorks(cacheEntry->decisionWorks); + result->emplace(std::move(cacheEntry->debugInfo), + std::make_pair(std::move(root), std::move(stageData))); + return result; } + // A temporary function to allow recovering SBE plans from the classic plan cache. + // TODO SERVER-61314: Remove this function when "featureFlagSbePlanCache" is removed. + std::unique_ptr<SlotBasedPrepareExecutionResult> buildCachedPlanFromClassicCache( + const QueryPlannerParams& plannerParams) { + auto planCacheKey = plan_cache_key_factory::make<PlanCacheKey>(*_cq, _collection); + OpDebug& opDebug = CurOp::get(_opCtx)->debug(); + if (!opDebug.planCacheKey) { + opDebug.planCacheKey = planCacheKey.planCacheKeyHash(); + } + // Try to look up a cached solution for the query. + if (auto cs = CollectionQueryInfo::get(_collection) + .getPlanCache() + ->getCacheEntryIfActive(planCacheKey)) { + // We have a CachedSolution. Have the planner turn it into a QuerySolution. + auto statusWithQs = QueryPlanner::planFromCache(*_cq, plannerParams, *cs); + + if (statusWithQs.isOK()) { + auto querySolution = std::move(statusWithQs.getValue()); + if ((plannerParams.options & QueryPlannerParams::IS_COUNT) && + turnIxscanIntoCount(querySolution.get())) { + LOGV2_DEBUG( + 20923, 2, "Using fast count", "query"_attr = redact(_cq->toStringShort())); + } + + auto result = makeResult(); + auto&& execTree = buildExecutableTree(*querySolution); + + result->emplace(std::move(execTree), std::move(querySolution)); + result->setDecisionWorks(cs->decisionWorks); + + return result; + } + } + + return nullptr; + } + std::unique_ptr<SlotBasedPrepareExecutionResult> buildSubPlan( const QueryPlannerParams& plannerParams) final { // Nothing to be done here, all planning and stage building will be done by a SubPlanner. @@ -1025,8 +1109,8 @@ std::unique_ptr<sbe::RuntimePlanner> makeRuntimePlannerIfNeeded( boost::optional<size_t> decisionWorks, bool needsSubplanning, PlanYieldPolicySBE* yieldPolicy, - size_t plannerOptions) { - + size_t plannerOptions, + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> debugInfo) { // If we have multiple solutions, we always need to do the runtime planning. if (numSolutions > 1) { invariant(!needsSubplanning && !decisionWorks); @@ -1059,8 +1143,13 @@ std::unique_ptr<sbe::RuntimePlanner> makeRuntimePlannerIfNeeded( plannerParams.options = plannerOptions; fillOutPlannerParams(opCtx, collection, canonicalQuery, &plannerParams); - return std::make_unique<sbe::CachedSolutionPlanner>( - opCtx, collection, *canonicalQuery, plannerParams, *decisionWorks, yieldPolicy); + return std::make_unique<sbe::CachedSolutionPlanner>(opCtx, + collection, + *canonicalQuery, + plannerParams, + *decisionWorks, + yieldPolicy, + std::move(debugInfo)); } // Runtime planning is not required. @@ -1106,8 +1195,9 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExe if (!planningResultWithStatus.isOK()) { return planningResultWithStatus.getStatus(); } + auto&& planningResult = planningResultWithStatus.getValue(); - auto&& [roots, solutions] = planningResult->extractResultData(); + auto&& [roots, solutions, debugInfo] = planningResult->extractResultData(); // In some circumstances (e.g. when have multiple candidate plans or using a cached one), we // might need to execute the plan(s) to pick the best one or to confirm the choice. @@ -1118,7 +1208,8 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExe planningResult->decisionWorks(), planningResult->needsSubplanning(), yieldPolicy.get(), - plannerOptions)) { + plannerOptions, + std::move(debugInfo))) { // Do the runtime planning and pick the best candidate plan. auto candidates = planner->plan(std::move(solutions), std::move(roots)); diff --git a/src/mongo/db/query/plan_cache.h b/src/mongo/db/query/plan_cache.h index 3623f8cb734..64afadd9834 100644 --- a/src/mongo/db/query/plan_cache.h +++ b/src/mongo/db/query/plan_cache.h @@ -40,21 +40,24 @@ namespace mongo { class QuerySolution; struct QuerySolutionNode; -template <class CachedPlanType> +template <class CachedPlanType, class DebugInfoType> class PlanCacheEntryBase; /** * Information returned from a get(...) query. */ -template <class CachedPlanType> +template <class CachedPlanType, class DebugInfoType> class CachedPlanHolder { private: CachedPlanHolder(const CachedPlanHolder&) = delete; CachedPlanHolder& operator=(const CachedPlanHolder&) = delete; public: - CachedPlanHolder(const PlanCacheEntryBase<CachedPlanType>& entry) - : cachedPlan(entry.cachedPlan->clone()), decisionWorks(entry.works) {} + CachedPlanHolder(const PlanCacheEntryBase<CachedPlanType, DebugInfoType>& entry) + : cachedPlan(entry.cachedPlan->clone()), + decisionWorks(entry.works), + debugInfo(entry.debugInfo ? std::make_unique<DebugInfoType>(*entry.debugInfo) : nullptr) { + } // A cached plan that can be used to reconstitute the complete execution plan from cache. @@ -63,48 +66,54 @@ public: // The number of work cycles taken to decide on a winning plan when the plan was first // cached. const size_t decisionWorks; + + // Per-plan cache entry information that is used for debugging purpose. + std::unique_ptr<DebugInfoType> debugInfo; }; /** * Used by the cache to track entries and their performance over time. * Also used by the plan cache commands to display plan cache state. */ -template <class CachedPlanType> +template <class CachedPlanType, class DebugInfoType> class PlanCacheEntryBase { public: - template <typename KeyType> - static std::unique_ptr<PlanCacheEntryBase<CachedPlanType>> create( - std::unique_ptr<const plan_ranker::PlanRankingDecision> decision, - std::unique_ptr<CachedPlanType> cachedPlan, - uint32_t queryHash, - uint32_t planCacheKey, - Date_t timeOfCreation, - bool isActive, - size_t works, - const PlanCacheCallbacks<KeyType, CachedPlanType>* callbacks) { - invariant(decision); + using Entry = PlanCacheEntryBase<CachedPlanType, DebugInfoType>; + + static std::unique_ptr<Entry> create(std::unique_ptr<CachedPlanType> cachedPlan, + uint32_t queryHash, + uint32_t planCacheKey, + Date_t timeOfCreation, + bool isActive, + size_t works, + DebugInfoType debugInfo) { // If the cumulative size of the plan caches is estimated to remain within a predefined // threshold, then then include additional debug info which is not strictly necessary for // the plan cache to be functional. Once the cumulative plan cache size exceeds this // threshold, omit this debug info as a heuristic to prevent plan cache memory consumption // from growing too large. - const bool includeDebugInfo = planCacheTotalSizeEstimateBytes.get() < + bool includeDebugInfo = planCacheTotalSizeEstimateBytes.get() < internalQueryCacheMaxSizeBytesBeforeStripDebugInfo.load(); - boost::optional<plan_cache_debug_info::DebugInfo> debugInfo; - if (includeDebugInfo && callbacks) { - debugInfo.emplace(callbacks->buildDebugInfo(std::move(decision))); + // The stripping logic does not apply to SBE's debugging info as "DebugInfoSBE" is not + // expected to be huge and is required to build a PlanExplainerSBE for the executor. + if constexpr (std::is_same_v<DebugInfoType, plan_cache_debug_info::DebugInfoSBE>) { + includeDebugInfo = true; } - return std::unique_ptr<PlanCacheEntryBase<CachedPlanType>>( - new PlanCacheEntryBase<CachedPlanType>(std::move(cachedPlan), - timeOfCreation, - queryHash, - planCacheKey, - isActive, - works, - std::move(debugInfo))); + boost::optional<DebugInfoType> debugInfoOpt; + if (includeDebugInfo) { + debugInfoOpt.emplace(std::move(debugInfo)); + } + + return std::unique_ptr<Entry>(new Entry(std::move(cachedPlan), + timeOfCreation, + queryHash, + planCacheKey, + isActive, + works, + std::move(debugInfoOpt))); } ~PlanCacheEntryBase() { @@ -114,20 +123,19 @@ public: /** * Make a deep copy. */ - std::unique_ptr<PlanCacheEntryBase<CachedPlanType>> clone() const { - boost::optional<plan_cache_debug_info::DebugInfo> debugInfoCopy; + std::unique_ptr<Entry> clone() const { + boost::optional<DebugInfoType> debugInfoCopy; if (debugInfo) { debugInfoCopy.emplace(*debugInfo); } - return std::unique_ptr<PlanCacheEntryBase<CachedPlanType>>( - new PlanCacheEntryBase<CachedPlanType>(cachedPlan->clone(), - timeOfCreation, - queryHash, - planCacheKey, - isActive, - works, - std::move(debugInfoCopy))); + return std::unique_ptr<Entry>(new Entry(cachedPlan->clone(), + timeOfCreation, + queryHash, + planCacheKey, + isActive, + works, + std::move(debugInfoCopy))); } std::string debugString() const { @@ -136,8 +144,7 @@ public: builder << "queryHash: " << queryHash; builder << "; planCacheKey: " << planCacheKey; if (debugInfo) { - builder << "; "; - builder << debugInfo->createdFromQuery.debugString(); + builder << "; " << debugInfo->debugString(); } builder << "; timeOfCreation: " << timeOfCreation.toString() << ")"; return builder.str(); @@ -165,13 +172,9 @@ public: // cause this value to be increased. size_t works = 0; - // Optional debug info containing detailed statistics. Includes a description of the query which - // resulted in this plan cache's creation as well as runtime stats from the multi-planner trial - // period that resulted in this cache entry. - // - // Once the estimated cumulative size of the mongod's plan caches exceeds a threshold, this - // debug info is omitted from new plan cache entries. - const boost::optional<plan_cache_debug_info::DebugInfo> debugInfo; + // Optional debug info containing plan cache entry information that is used strictly as + // debug information. + const boost::optional<DebugInfoType> debugInfo; // An estimate of the size in bytes of this plan cache entry. This is the "deep size", // calculated by recursively incorporating the size of owned objects, the objects that they in @@ -193,7 +196,7 @@ private: uint32_t planCacheKey, bool isActive, size_t works, - boost::optional<plan_cache_debug_info::DebugInfo> debugInfo) + boost::optional<DebugInfoType> debugInfo) : cachedPlan(std::move(cachedPlan)), timeOfCreation(timeOfCreation), queryHash(queryHash), @@ -234,6 +237,7 @@ private: template <class KeyType, class CachedPlanType, class BudgetEstimator, + class DebugInfoType, class Partitioner, class KeyHasher = std::hash<KeyType>> class PlanCacheBase { @@ -242,7 +246,7 @@ private: PlanCacheBase& operator=(const PlanCacheBase&) = delete; public: - using Entry = PlanCacheEntryBase<CachedPlanType>; + using Entry = PlanCacheEntryBase<CachedPlanType, DebugInfoType>; using Lru = LRUKeyValue<KeyType, Entry, BudgetEstimator, KeyHasher>; // We have three states for a cache entry to be in. Rather than just 'present' or 'not @@ -269,7 +273,7 @@ public: */ struct GetResult { CacheEntryState state; - std::unique_ptr<CachedPlanHolder<CachedPlanType>> cachedPlanHolder; + std::unique_ptr<CachedPlanHolder<CachedPlanType, DebugInfoType>> cachedPlanHolder; }; /** @@ -300,16 +304,17 @@ public: * * If the mapping was set successfully, returns Status::OK(), even if it evicted another entry. */ - Status set(const KeyType& key, - std::unique_ptr<CachedPlanType> cachedPlan, - std::unique_ptr<plan_ranker::PlanRankingDecision> why, - Date_t now, - boost::optional<double> worksGrowthCoefficient = boost::none, - const PlanCacheCallbacks<KeyType, CachedPlanType>* callbacks = nullptr) { - invariant(why); + Status set( + const KeyType& key, + std::unique_ptr<CachedPlanType> cachedPlan, + const plan_ranker::PlanRankingDecision& why, + Date_t now, + DebugInfoType debugInfo, + boost::optional<double> worksGrowthCoefficient = boost::none, + const PlanCacheCallbacks<KeyType, CachedPlanType, DebugInfoType>* callbacks = nullptr) { invariant(cachedPlan); - if (why->scores.size() != why->candidateOrder.size()) { + if (why.scores.size() != why.candidateOrder.size()) { return Status(ErrorCodes::BadValue, "number of scores in decision must match viable candidates"); } @@ -322,7 +327,7 @@ public: return calculateNumberOfReads( details.candidatePlanStats[0].get()); }}, - why->stats); + why.stats); auto partition = _partitionedCache->lockOnePartition(key); auto [queryHash, planCacheKey, isNewEntryActive, shouldBeCreated] = [&]() { @@ -364,14 +369,13 @@ public: return Status::OK(); } - auto newEntry(Entry::create(std::move(why), - std::move(cachedPlan), + auto newEntry(Entry::create(std::move(cachedPlan), queryHash, planCacheKey, now, isNewEntryActive, newWorks, - callbacks)); + std::move(debugInfo))); partition->add(key, newEntry.release()); return Status::OK(); @@ -420,14 +424,16 @@ public: auto state = entry.getValue()->isActive ? CacheEntryState::kPresentActive : CacheEntryState::kPresentInactive; - return {state, std::make_unique<CachedPlanHolder<CachedPlanType>>(*entry.getValue())}; + return { + state, + std::make_unique<CachedPlanHolder<CachedPlanType, DebugInfoType>>(*entry.getValue())}; } /** * If the cache entry exists and is active, return a CachedSolution. If the cache entry is * inactive, log a message and return a nullptr. If no cache entry exists, return a nullptr. */ - std::unique_ptr<CachedPlanHolder<CachedPlanType>> getCacheEntryIfActive( + std::unique_ptr<CachedPlanHolder<CachedPlanType, DebugInfoType>> getCacheEntryIfActive( const KeyType& key) const { auto res = get(key); if (res.state == CacheEntryState::kPresentInactive) { @@ -446,7 +452,6 @@ public: _partitionedCache->erase(key); } - /** * Remove all the entries for keys for which the predicate returns true. Return the number of * removed entries. @@ -567,11 +572,12 @@ private: * - We should create a new entry * - The new entry should be marked 'active' */ - NewEntryState getNewEntryState(const KeyType& key, - Entry* oldEntry, - size_t newWorks, - double growthCoefficient, - const PlanCacheCallbacks<KeyType, CachedPlanType>* callbacks) { + NewEntryState getNewEntryState( + const KeyType& key, + Entry* oldEntry, + size_t newWorks, + double growthCoefficient, + const PlanCacheCallbacks<KeyType, CachedPlanType, DebugInfoType>* callbacks) { NewEntryState res; if (!oldEntry) { if (callbacks) { diff --git a/src/mongo/db/query/plan_cache_callbacks.h b/src/mongo/db/query/plan_cache_callbacks.h index e4e58550c41..0950b2653ce 100644 --- a/src/mongo/db/query/plan_cache_callbacks.h +++ b/src/mongo/db/query/plan_cache_callbacks.h @@ -65,56 +65,63 @@ void logPromoteCacheEntry(std::string&& query, size_t newWorks); } // namespace log_detail -template <class CachedPlanType> +template <class CachedPlanType, class DebugInfo> class PlanCacheEntryBase; +struct SolutionCacheData; /** * Encapsulates callback functions used to perform a custom action when the plan cache state * changes. */ -template <typename KeyType, typename CachedPlanType> +template <typename KeyType, typename CachedPlanType, typename DebugInfoType> class PlanCacheCallbacks { public: virtual ~PlanCacheCallbacks() = default; - virtual void onCreateInactiveCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, - size_t newWorks) const = 0; - virtual void onReplaceActiveCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, - size_t newWorks) const = 0; - virtual void onNoopActiveCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, - size_t newWorks) const = 0; - virtual void onIncreasingWorkValue(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, - size_t newWorks) const = 0; - virtual void onPromoteCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, - size_t newWorks) const = 0; - virtual plan_cache_debug_info::DebugInfo buildDebugInfo( - std::unique_ptr<const plan_ranker::PlanRankingDecision> decision) const = 0; + virtual void onCreateInactiveCacheEntry( + const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, + size_t newWorks) const = 0; + virtual void onReplaceActiveCacheEntry( + const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, + size_t newWorks) const = 0; + virtual void onNoopActiveCacheEntry( + const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, + size_t newWorks) const = 0; + virtual void onIncreasingWorkValue( + const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, + size_t newWorks) const = 0; + virtual void onPromoteCacheEntry( + const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, + size_t newWorks) const = 0; }; /** * Simple logging callbacks for the plan cache. */ -template <typename KeyType, typename CachedPlanType> -class PlanCacheLoggingCallbacks : public PlanCacheCallbacks<KeyType, CachedPlanType> { +template <typename KeyType, typename CachedPlanType, typename DebugInfoType> +class PlanCacheLoggingCallbacks + : public PlanCacheCallbacks<KeyType, CachedPlanType, DebugInfoType> { public: PlanCacheLoggingCallbacks(const CanonicalQuery& cq) : _cq{cq} {} - void onCreateInactiveCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, - size_t newWorks) const final { + void onCreateInactiveCacheEntry( + const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, + size_t newWorks) const final { auto&& [queryHash, planCacheKey] = hashes(key, oldEntry); log_detail::logCreateInactiveCacheEntry( _cq.toStringShort(), std::move(queryHash), std::move(planCacheKey), newWorks); } - void onReplaceActiveCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, - size_t newWorks) const final { + void onReplaceActiveCacheEntry( + const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, + size_t newWorks) const final { invariant(oldEntry); auto&& [queryHash, planCacheKey] = hashes(key, oldEntry); log_detail::logReplaceActiveCacheEntry(_cq.toStringShort(), @@ -125,7 +132,7 @@ public: } void onNoopActiveCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, size_t newWorks) const final { invariant(oldEntry); auto&& [queryHash, planCacheKey] = hashes(key, oldEntry); @@ -137,7 +144,7 @@ public: } void onIncreasingWorkValue(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, size_t newWorks) const final { invariant(oldEntry); auto&& [queryHash, planCacheKey] = hashes(key, oldEntry); @@ -149,7 +156,7 @@ public: } void onPromoteCacheEntry(const KeyType& key, - const PlanCacheEntryBase<CachedPlanType>* oldEntry, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry, size_t newWorks) const final { invariant(oldEntry); auto&& [queryHash, planCacheKey] = hashes(key, oldEntry); @@ -160,13 +167,9 @@ public: newWorks); } - plan_cache_debug_info::DebugInfo buildDebugInfo( - std::unique_ptr<const plan_ranker::PlanRankingDecision> decision) const final { - return plan_cache_debug_info::buildDebugInfo(_cq, std::move(decision)); - } - private: - auto hashes(const KeyType& key, const PlanCacheEntryBase<CachedPlanType>* oldEntry) const { + auto hashes(const KeyType& key, + const PlanCacheEntryBase<CachedPlanType, DebugInfoType>* oldEntry) const { // Avoid recomputing the hashes if we've got an old entry to grab them from. return oldEntry ? std::make_pair(zeroPaddedHex(oldEntry->queryHash), diff --git a/src/mongo/db/query/plan_cache_debug_info.cpp b/src/mongo/db/query/plan_cache_debug_info.cpp deleted file mode 100644 index 4b7651e95db..00000000000 --- a/src/mongo/db/query/plan_cache_debug_info.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright (C) 2021-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/query/plan_cache_debug_info.h" - -namespace mongo::plan_cache_debug_info { -DebugInfo buildDebugInfo(const CanonicalQuery& query, - std::unique_ptr<const plan_ranker::PlanRankingDecision> decision) { - // Strip projections on $-prefixed fields, as these are added by internal callers of the - // system and are not considered part of the user projection. - const FindCommandRequest& findCommand = query.getFindCommandRequest(); - BSONObjBuilder projBuilder; - for (auto elem : findCommand.getProjection()) { - if (elem.fieldName()[0] == '$') { - continue; - } - projBuilder.append(elem); - } - - CreatedFromQuery createdFromQuery{findCommand.getFilter(), - findCommand.getSort(), - projBuilder.obj(), - query.getCollator() ? query.getCollator()->getSpec().toBSON() - : BSONObj()}; - - return {std::move(createdFromQuery), std::move(decision)}; -} -} // namespace mongo::plan_cache_debug_info diff --git a/src/mongo/db/query/plan_cache_debug_info.h b/src/mongo/db/query/plan_cache_debug_info.h index 611e947c234..d7824ce7a67 100644 --- a/src/mongo/db/query/plan_cache_debug_info.h +++ b/src/mongo/db/query/plan_cache_debug_info.h @@ -31,6 +31,7 @@ #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/plan_ranking_decision.h" +#include "mongo/util/container_size_helper.h" namespace mongo::plan_cache_debug_info { /** @@ -105,6 +106,10 @@ struct DebugInfo { return size; } + std::string debugString() const { + return createdFromQuery.debugString(); + } + CreatedFromQuery createdFromQuery; // Information that went into picking the winning plan and also why the other plans lost. @@ -112,6 +117,23 @@ struct DebugInfo { std::unique_ptr<const plan_ranker::PlanRankingDecision> decision; }; -DebugInfo buildDebugInfo(const CanonicalQuery& query, - std::unique_ptr<const plan_ranker::PlanRankingDecision> decision); +/* + * Similar to "DebugInfo" above. This debug info struct is only for SBE plan cache. + */ +struct DebugInfoSBE { + uint64_t estimateObjectSizeInBytes() const { + return sizeof(DebugInfoSBE) + planSummary.capacity() + + container_size_helper::estimateObjectSizeInBytes( + indexesUsed, [](std::string str) { return str.capacity(); }, true); + } + + std::string debugString() const { + return planSummary; + } + + long long collectionScans = 0; + long long collectionScansNonTailable = 0; + std::string planSummary; + std::vector<std::string> indexesUsed; +}; } // namespace mongo::plan_cache_debug_info diff --git a/src/mongo/db/query/plan_cache_test.cpp b/src/mongo/db/query/plan_cache_test.cpp index ecf4c2a676e..fc48a792e0e 100644 --- a/src/mongo/db/query/plan_cache_test.cpp +++ b/src/mongo/db/query/plan_cache_test.cpp @@ -39,6 +39,7 @@ #include <memory> #include <ostream> +#include "mongo/db/exec/plan_cache_util.h" #include "mongo/db/index/wildcard_key_generator.h" #include "mongo/db/jsobj.h" #include "mongo/db/json.h" @@ -308,7 +309,13 @@ void addCacheEntryForShape(const CanonicalQuery& cq, PlanCache* planCache) { invariant(planCache); auto qs = getQuerySolutionForCaching(); - ASSERT_OK(planCache->set(makeKey(cq), qs->cacheData->clone(), createDecision(1U), Date_t{})); + auto decision = createDecision(1U); + auto decisionPtr = decision.get(); + ASSERT_OK(planCache->set(makeKey(cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(cq, std::move(decision)))); } TEST(PlanCacheTest, InactiveEntriesDisabled) { @@ -320,10 +327,16 @@ TEST(PlanCacheTest, InactiveEntriesDisabled) { unique_ptr<CanonicalQuery> cq(canonicalize("{a: 1}")); auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U); + auto decisionPtr = decision.get(); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an _active_ entry. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentActive); @@ -382,9 +395,15 @@ TEST(PlanCacheTest, PlanCacheRemoveDeletesInactiveEntries) { auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U); + auto decisionPtr = decision.get(); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an inactive entry. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); @@ -401,9 +420,15 @@ TEST(PlanCacheTest, PlanCacheFlushDeletesInactiveEntries) { auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U); + auto decisionPtr = decision.get(); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an inactive entry. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); @@ -420,17 +445,29 @@ TEST(PlanCacheTest, AddActiveCacheEntry) { auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U, 20); + auto decisionPtr = decision.get(); // Check if key is in cache before and after set(). ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 20), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an inactive entry. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); + decision = createDecision(1U, 10); + decisionPtr = decision.get(); // Calling set() again, with a solution that had a lower works value should create an active // entry. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 10), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentActive); ASSERT_EQUALS(planCache.size(), 1U); @@ -445,14 +482,20 @@ TEST(PlanCacheTest, WorksValueIncreases) { unique_ptr<CanonicalQuery> cq(canonicalize("{a: 1}")); auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*cq}; + auto decisionPtr = createDecision(1U, 10); + auto decision = decisionPtr.get(); + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + mongo::plan_cache_debug_info::DebugInfo> + callbacks{*cq}; ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(1U, 10), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); @@ -462,9 +505,15 @@ TEST(PlanCacheTest, WorksValueIncreases) { ASSERT_EQ(entry->works, 10U); ASSERT_FALSE(entry->isActive); + decisionPtr = createDecision(1U, 50); + decision = decisionPtr.get(); // Calling set() again, with a solution that had a higher works value. This should cause the // works on the original entry to be increased. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 50), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decision, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)))); // The entry should still be inactive. Its works should double though. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); @@ -472,9 +521,15 @@ TEST(PlanCacheTest, WorksValueIncreases) { ASSERT_FALSE(entry->isActive); ASSERT_EQ(entry->works, 20U); + decisionPtr = createDecision(1U, 30); + decision = decisionPtr.get(); // Calling set() again, with a solution that had a higher works value. This should cause the // works on the original entry to be increased. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 30), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decision, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)))); // The entry should still be inactive. Its works should have doubled again. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); @@ -482,12 +537,15 @@ TEST(PlanCacheTest, WorksValueIncreases) { ASSERT_FALSE(entry->isActive); ASSERT_EQ(entry->works, 40U); + decisionPtr = createDecision(1U, 25); + decision = decisionPtr.get(); // Calling set() again, with a solution that has a lower works value than what's currently in // the cache. ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(1U, 25), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); @@ -499,8 +557,8 @@ TEST(PlanCacheTest, WorksValueIncreases) { ASSERT(entry->debugInfo); ASSERT(entry->debugInfo->decision); - auto&& decision = entry->debugInfo->decision; - ASSERT_EQ(decision->getStats<PlanStageStats>().candidatePlanStats[0]->common.works, 25U); + auto&& decision1 = entry->debugInfo->decision; + ASSERT_EQ(decision1->getStats<PlanStageStats>().candidatePlanStats[0]->common.works, 25U); ASSERT_EQ(entry->works, 25U); ASSERT_EQUALS(planCache.size(), 1U); @@ -519,10 +577,16 @@ TEST(PlanCacheTest, WorksValueIncreasesByAtLeastOne) { unique_ptr<CanonicalQuery> cq(canonicalize("{a: 1}")); auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U, 3); + auto decisionPtr = decision.get(); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 3), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an inactive entry. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); @@ -530,12 +594,18 @@ TEST(PlanCacheTest, WorksValueIncreasesByAtLeastOne) { ASSERT_EQ(entry->works, 3U); ASSERT_FALSE(entry->isActive); + decision = createDecision(1U, 50); + decisionPtr = decision.get(); // Calling set() again, with a solution that had a higher works value. This should cause the // works on the original entry to be increased. In this case, since nWorks is 3, // multiplying by the value 1.10 will give a value of 3 (static_cast<size_t>(1.1 * 3) == 3). // We check that the works value is increased 1 instead. - ASSERT_OK( - planCache.set(key, qs->cacheData->clone(), createDecision(1U, 50), Date_t{}, kWorksCoeff)); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)), + kWorksCoeff)); // The entry should still be inactive. Its works should increase by 1. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); @@ -554,10 +624,16 @@ TEST(PlanCacheTest, SetIsNoopWhenNewEntryIsWorse) { unique_ptr<CanonicalQuery> cq(canonicalize("{a: 1}")); auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U, 50); + auto decisionPtr = decision.get(); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 50), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an inactive entry. ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentInactive); @@ -565,17 +641,29 @@ TEST(PlanCacheTest, SetIsNoopWhenNewEntryIsWorse) { ASSERT_EQ(entry->works, 50U); ASSERT_FALSE(entry->isActive); + decision = createDecision(1U, 20); + decisionPtr = decision.get(); // Call set() again, with a solution that has a lower works value. This will result in an // active entry being created. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 20), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentActive); entry = assertGet(planCache.getEntry(key)); ASSERT_TRUE(entry->isActive); ASSERT_EQ(entry->works, 20U); + decision = createDecision(1U, 100); + decisionPtr = decision.get(); // Now call set() again, but with a solution that has a higher works value. This should be // a noop. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 100), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentActive); entry = assertGet(planCache.getEntry(key)); ASSERT_TRUE(entry->isActive); @@ -588,26 +676,44 @@ TEST(PlanCacheTest, SetOverwritesWhenNewEntryIsBetter) { auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U, 50); + auto decisionPtr = decision.get(); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 50), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an inactive entry. auto entry = assertGet(planCache.getEntry(key)); ASSERT_EQ(entry->works, 50U); ASSERT_FALSE(entry->isActive); + decision = createDecision(1U, 20); + decisionPtr = decision.get(); // Call set() again, with a solution that has a lower works value. This will result in an // active entry being created. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 20), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentActive); entry = assertGet(planCache.getEntry(key)); ASSERT_TRUE(entry->isActive); ASSERT_EQ(entry->works, 20U); + decision = createDecision(1U, 10); + decisionPtr = decision.get(); // Now call set() again, with a solution that has a lower works value. The current active entry // should be overwritten. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 10), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentActive); entry = assertGet(planCache.getEntry(key)); ASSERT_TRUE(entry->isActive); @@ -619,19 +725,31 @@ TEST(PlanCacheTest, DeactivateCacheEntry) { unique_ptr<CanonicalQuery> cq(canonicalize("{a: 1}")); auto qs = getQuerySolutionForCaching(); auto key = makeKey(*cq); + auto decision = createDecision(1U, 50); + auto decisionPtr = decision.get(); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kNotPresent); QueryTestServiceContext serviceContext; - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 50), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); // After add, the planCache should have an inactive entry. auto entry = assertGet(planCache.getEntry(key)); ASSERT_EQ(entry->works, 50U); ASSERT_FALSE(entry->isActive); + decision = createDecision(1U, 20); + decisionPtr = decision.get(); // Call set() again, with a solution that has a lower works value. This will result in an // active entry being created. - ASSERT_OK(planCache.set(key, qs->cacheData->clone(), createDecision(1U, 20), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); ASSERT_EQ(planCache.get(key).state, PlanCache::CacheEntryState::kPresentActive); entry = assertGet(planCache.getEntry(key)); ASSERT_TRUE(entry->isActive); @@ -653,16 +771,26 @@ TEST(PlanCacheTest, GetMatchingStatsMatchesAndSerializesCorrectly) { { unique_ptr<CanonicalQuery> cq(canonicalize("{a: 1}")); auto qs = getQuerySolutionForCaching(); - ASSERT_OK( - planCache.set(makeKey(*cq), qs->cacheData->clone(), createDecision(1U, 5), Date_t{})); + auto decision = createDecision(1U, 5); + auto decisionPtr = decision.get(); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); } // Create a second cache entry with 3 works. { + auto decision = createDecision(1U, 3); unique_ptr<CanonicalQuery> cq(canonicalize("{b: 1}")); auto qs = getQuerySolutionForCaching(); - ASSERT_OK( - planCache.set(makeKey(*cq), qs->cacheData->clone(), createDecision(1U, 3), Date_t{})); + auto decisionPtr = decision.get(); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); } // Verify that the cache entries have been created. @@ -966,14 +1094,15 @@ protected: uint32_t queryHash = ck.queryHash(); uint32_t planCacheKey = queryHash; - auto entry = PlanCacheEntry::create<PlanCacheKey>(createDecision(1U), - qs.cacheData->clone(), - queryHash, - planCacheKey, - Date_t(), - false /* isActive */, - 0 /* works */, - nullptr /* callbacks */); + auto decision = createDecision(1U); + auto entry = + PlanCacheEntry::create(qs.cacheData->clone(), + queryHash, + planCacheKey, + Date_t(), + false /* isActive */, + 0 /* works */, + plan_cache_util::buildDebugInfo(*scopedCq, std::move(decision))); CachedSolution cachedSoln(*entry); auto statusWithQs = QueryPlanner::planFromCache(*scopedCq, params, cachedSoln); @@ -1651,43 +1780,58 @@ TEST(PlanCacheTest, PlanCacheSizeWithCRUDOperations) { auto qs = getQuerySolutionForCaching(); long long previousSize, originalSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); auto key = makeKey(*cq); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*cq}; + auto decisionPtr = createDecision(1U); + auto decision = decisionPtr.get(); + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + mongo::plan_cache_debug_info::DebugInfo> + callbacks{*cq}; // Verify that the plan cache size increases after adding new entry to cache. previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(1U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); + decisionPtr = createDecision(1U); + decision = decisionPtr.get(); // Verify that trying to set the same entry won't change the plan cache size. previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(1U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_EQ(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); + decisionPtr = createDecision(2U); + decision = decisionPtr.get(); // Verify that the plan cache size increases after updating the same entry with more solutions. ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(2U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); + decisionPtr = createDecision(1U); + decision = decisionPtr.get(); // Verify that the plan cache size decreases after updating the same entry with fewer solutions. previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(1U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_LT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); @@ -1697,14 +1841,17 @@ TEST(PlanCacheTest, PlanCacheSizeWithCRUDOperations) { long long sizeWithOneEntry = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); std::string queryString = "{a: 1, c: 1}"; for (int i = 0; i < 5; ++i) { + decisionPtr = createDecision(1U); + decision = decisionPtr.get(); // Update the field name in the query string so that plan cache creates a new entry. queryString[1] = 'b' + i; unique_ptr<CanonicalQuery> query(canonicalize(queryString)); previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); ASSERT_OK(planCache.set(makeKey(*query), qs->cacheData->clone(), - createDecision(1U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); @@ -1746,15 +1893,21 @@ TEST(PlanCacheTest, PlanCacheSizeWithEviction) { // Add entries until plan cache is full and verify that the size keeps increasing. std::string queryString = "{a: 1, c: 1}"; for (size_t i = 0; i < kCacheSize; ++i) { + auto decisionPtr = createDecision(2U); + auto decision = decisionPtr.get(); // Update the field name in the query string so that plan cache creates a new entry. queryString[1]++; unique_ptr<CanonicalQuery> query(canonicalize(queryString)); previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*cq}; + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + mongo::plan_cache_debug_info::DebugInfo> + callbacks{*cq}; ASSERT_OK(planCache.set(makeKey(*query), qs->cacheData->clone(), - createDecision(2U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); @@ -1762,15 +1915,21 @@ TEST(PlanCacheTest, PlanCacheSizeWithEviction) { // Verify that adding entry of same size as evicted entry wouldn't change the plan cache size. { + auto decisionPtr = createDecision(2U); + auto decision = decisionPtr.get(); queryString = "{k: 1, c: 1}"; cq = unique_ptr<CanonicalQuery>(canonicalize(queryString)); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*cq}; + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + mongo::plan_cache_debug_info::DebugInfo> + callbacks{*cq}; previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); ASSERT_EQ(planCache.size(), kCacheSize); ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(2U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_EQ(planCache.size(), kCacheSize); @@ -1780,30 +1939,43 @@ TEST(PlanCacheTest, PlanCacheSizeWithEviction) { // Verify that adding entry with query bigger than the evicted entry's key should change the // plan cache size. { + auto decisionPtr = createDecision(2U); + auto decision = decisionPtr.get(); queryString = "{k: 1, c: 1, extraField: 1}"; unique_ptr<CanonicalQuery> queryBiggerKey(canonicalize(queryString)); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*queryBiggerKey}; + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + mongo::plan_cache_debug_info::DebugInfo> + callbacks{*queryBiggerKey}; previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); - ASSERT_OK(planCache.set(makeKey(*queryBiggerKey), - qs->cacheData->clone(), - createDecision(2U), - Date_t{}, - boost::none /* worksGrowthCoefficient */, - &callbacks)); + ASSERT_OK( + planCache.set(makeKey(*queryBiggerKey), + qs->cacheData->clone(), + *decision, + Date_t{}, + plan_cache_util::buildDebugInfo(*queryBiggerKey, std::move(decisionPtr)), + boost::none /* worksGrowthCoefficient */, + &callbacks)); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); } // Verify that adding entry with query solutions larger than the evicted entry's query solutions // should increase the plan cache size. { + auto decisionPtr = createDecision(3U); + auto decision = decisionPtr.get(); queryString = "{l: 1, c: 1}"; cq = unique_ptr<CanonicalQuery>(canonicalize(queryString)); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*cq}; + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + mongo::plan_cache_debug_info::DebugInfo> + callbacks{*cq}; previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(3U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); @@ -1812,14 +1984,20 @@ TEST(PlanCacheTest, PlanCacheSizeWithEviction) { // Verify that adding entry with query solutions smaller than the evicted entry's query // solutions should decrease the plan cache size. { + auto decisionPtr = createDecision(1U); + auto decision = decisionPtr.get(); queryString = "{m: 1, c: 1}"; cq = unique_ptr<CanonicalQuery>(canonicalize(queryString)); - PlanCacheLoggingCallbacks<PlanCacheKey, SolutionCacheData> callbacks{*cq}; + PlanCacheLoggingCallbacks<PlanCacheKey, + SolutionCacheData, + mongo::plan_cache_debug_info::DebugInfo> + callbacks{*cq}; previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); ASSERT_OK(planCache.set(key, qs->cacheData->clone(), - createDecision(1U), + *decision, Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decisionPtr)), boost::none /* worksGrowthCoefficient */, &callbacks)); ASSERT_LT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); @@ -1840,17 +2018,27 @@ TEST(PlanCacheTest, PlanCacheSizeWithMultiplePlanCaches) { // Verify that adding entries to both plan caches will keep increasing the cache size. std::string queryString = "{a: 1, c: 1}"; for (int i = 0; i < 5; ++i) { + auto decision = createDecision(1U); + auto decisionPtr = decision.get(); // Update the field name in the query string so that plan cache creates a new entry. queryString[1] = 'b' + i; unique_ptr<CanonicalQuery> query(canonicalize(queryString)); previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); - ASSERT_OK( - planCache1.set(makeKey(*query), qs->cacheData->clone(), createDecision(1U), Date_t{})); + ASSERT_OK(planCache1.set(makeKey(*query), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*query, std::move(decision)))); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); + decision = createDecision(1U); + decisionPtr = decision.get(); previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); - ASSERT_OK( - planCache2.set(makeKey(*query), qs->cacheData->clone(), createDecision(1U), Date_t{})); + ASSERT_OK(planCache2.set(makeKey(*query), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*query, std::move(decision)))); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); } @@ -1867,10 +2055,15 @@ TEST(PlanCacheTest, PlanCacheSizeWithMultiplePlanCaches) { // Verify for scoped PlanCache object. long long sizeBeforeScopedPlanCache = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); { + auto decision = createDecision(1U); + auto decisionPtr = decision.get(); PlanCache planCache(5000); previousSize = PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(); - ASSERT_OK( - planCache.set(makeKey(*cq), qs->cacheData->clone(), createDecision(1U), Date_t{})); + ASSERT_OK(planCache.set(makeKey(*cq), + qs->cacheData->clone(), + *decisionPtr, + Date_t{}, + plan_cache_util::buildDebugInfo(*cq, std::move(decision)))); ASSERT_GT(PlanCacheEntry::planCacheTotalSizeEstimateBytes.get(), previousSize); } diff --git a/src/mongo/db/query/plan_executor_sbe.cpp b/src/mongo/db/query/plan_executor_sbe.cpp index d15daede90e..3b7a2db857f 100644 --- a/src/mongo/db/query/plan_executor_sbe.cpp +++ b/src/mongo/db/query/plan_executor_sbe.cpp @@ -112,8 +112,12 @@ PlanExecutorSBE::PlanExecutorSBE(OperationContext* opCtx, candidates.plans.erase(candidates.plans.begin() + candidates.winnerIdx); } - _planExplainer = plan_explainer_factory::make( - _root.get(), &_rootData, _solution.get(), std::move(candidates.plans), isMultiPlan); + _planExplainer = plan_explainer_factory::make(_root.get(), + &_rootData, + _solution.get(), + std::move(candidates.plans), + isMultiPlan, + std::move(_rootData.debugInfo)); } void PlanExecutorSBE::saveState() { diff --git a/src/mongo/db/query/plan_explainer_factory.cpp b/src/mongo/db/query/plan_explainer_factory.cpp index 74d0c1dfd3d..ef4858e1f7c 100644 --- a/src/mongo/db/query/plan_explainer_factory.cpp +++ b/src/mongo/db/query/plan_explainer_factory.cpp @@ -31,6 +31,7 @@ #include "mongo/db/query/plan_explainer_factory.h" +#include "mongo/db/exec/plan_cache_util.h" #include "mongo/db/query/plan_explainer_impl.h" #include "mongo/db/query/plan_explainer_sbe.h" @@ -54,7 +55,29 @@ std::unique_ptr<PlanExplainer> make(sbe::PlanStage* root, const QuerySolution* solution, std::vector<sbe::plan_ranker::CandidatePlan> rejectedCandidates, bool isMultiPlan) { + // Pre-compute Debugging info for explain use. + auto debugInfoSBE = std::make_unique<plan_cache_debug_info::DebugInfoSBE>( + plan_cache_util::buildDebugInfo(solution)); return std::make_unique<PlanExplainerSBE>( - root, data, solution, std::move(rejectedCandidates), isMultiPlan); + root, data, solution, std::move(rejectedCandidates), isMultiPlan, std::move(debugInfoSBE)); +} + +std::unique_ptr<PlanExplainer> make( + sbe::PlanStage* root, + const stage_builder::PlanStageData* data, + const QuerySolution* solution, + std::vector<sbe::plan_ranker::CandidatePlan> rejectedCandidates, + bool isMultiPlan, + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> debugInfoSBE) { + // TODO SERVER-61314: Consider invariant(debugInfoSBE) as we may not need to create a + // DebugInfoSBE from QuerySolution after the feature flag is removed. We currently need it + // because debugInfoSBE can be null if the plan was recovered from the classic plan cache. + if (!debugInfoSBE) { + debugInfoSBE = std::make_unique<plan_cache_debug_info::DebugInfoSBE>( + plan_cache_util::buildDebugInfo(solution)); + } + + return std::make_unique<PlanExplainerSBE>( + root, data, solution, std::move(rejectedCandidates), isMultiPlan, std::move(debugInfoSBE)); } } // namespace mongo::plan_explainer_factory diff --git a/src/mongo/db/query/plan_explainer_factory.h b/src/mongo/db/query/plan_explainer_factory.h index f6736067bda..f0165f2c1cb 100644 --- a/src/mongo/db/query/plan_explainer_factory.h +++ b/src/mongo/db/query/plan_explainer_factory.h @@ -38,14 +38,24 @@ namespace mongo::plan_explainer_factory { std::unique_ptr<PlanExplainer> make(PlanStage* root); + std::unique_ptr<PlanExplainer> make(PlanStage* root, const PlanEnumeratorExplainInfo& enumeratorInfo); + std::unique_ptr<PlanExplainer> make(sbe::PlanStage* root, const stage_builder::PlanStageData* data, const QuerySolution* solution); + std::unique_ptr<PlanExplainer> make(sbe::PlanStage* root, const stage_builder::PlanStageData* data, const QuerySolution* solution, std::vector<sbe::plan_ranker::CandidatePlan> rejectedCandidates, bool isMultiPlan); + +std::unique_ptr<PlanExplainer> make(sbe::PlanStage* root, + const stage_builder::PlanStageData* data, + const QuerySolution* solution, + std::vector<sbe::plan_ranker::CandidatePlan> rejectedCandidates, + bool isMultiPlan, + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> debugInfo); } // namespace mongo::plan_explainer_factory diff --git a/src/mongo/db/query/plan_explainer_sbe.cpp b/src/mongo/db/query/plan_explainer_sbe.cpp index 20db0c44c32..8bde2311ab6 100644 --- a/src/mongo/db/query/plan_explainer_sbe.cpp +++ b/src/mongo/db/query/plan_explainer_sbe.cpp @@ -335,82 +335,13 @@ const PlanExplainer::ExplainVersion& PlanExplainerSBE::getVersion() const { } std::string PlanExplainerSBE::getPlanSummary() const { - if (!_solution) { - return {}; - } - - StringBuilder sb; - bool seenLeaf = false; - std::queue<const QuerySolutionNode*> queue; - queue.push(_solution->root()); - - while (!queue.empty()) { - auto node = queue.front(); - queue.pop(); - - if (node->children.empty()) { - if (seenLeaf) { - sb << ", "; - } else { - seenLeaf = true; - } - - sb << stageTypeToString(node->getType()); - - switch (node->getType()) { - case STAGE_COUNT_SCAN: { - auto csn = static_cast<const CountScanNode*>(node); - const KeyPattern keyPattern{csn->index.keyPattern}; - sb << " " << keyPattern; - break; - } - case STAGE_DISTINCT_SCAN: { - auto dn = static_cast<const DistinctNode*>(node); - const KeyPattern keyPattern{dn->index.keyPattern}; - sb << " " << keyPattern; - break; - } - case STAGE_GEO_NEAR_2D: { - auto geo2d = static_cast<const GeoNear2DNode*>(node); - const KeyPattern keyPattern{geo2d->index.keyPattern}; - sb << " " << keyPattern; - break; - } - case STAGE_GEO_NEAR_2DSPHERE: { - auto geo2dsphere = static_cast<const GeoNear2DSphereNode*>(node); - const KeyPattern keyPattern{geo2dsphere->index.keyPattern}; - sb << " " << keyPattern; - break; - } - case STAGE_IXSCAN: { - auto ixn = static_cast<const IndexScanNode*>(node); - const KeyPattern keyPattern{ixn->index.keyPattern}; - sb << " " << keyPattern; - break; - } - case STAGE_TEXT_MATCH: { - auto tn = static_cast<const TextMatchNode*>(node); - const KeyPattern keyPattern{tn->indexPrefix}; - sb << " " << keyPattern; - break; - } - default: - break; - } - } - - for (auto&& child : node->children) { - queue.push(child); - } - } - - return sb.str(); + return _debugInfo->planSummary; } void PlanExplainerSBE::getSummaryStats(PlanSummaryStats* statsOut) const { invariant(statsOut); - if (!_solution || !_root) { + if (!_root) { return; } @@ -428,61 +359,11 @@ void PlanExplainerSBE::getSummaryStats(PlanSummaryStats* statsOut) const { auto visitor = PlanSummaryStatsVisitor(*statsOut); _root->accumulate(kEmptyPlanNodeId, &visitor); - std::queue<const QuerySolutionNode*> queue; - queue.push(_solution->root()); - - // Look through the QuerySolution to collect some static stat details. - while (!queue.empty()) { - auto node = queue.front(); - queue.pop(); - invariant(node); - - switch (node->getType()) { - case STAGE_COUNT_SCAN: { - auto csn = static_cast<const CountScanNode*>(node); - statsOut->indexesUsed.insert(csn->index.identifier.catalogName); - break; - } - case STAGE_DISTINCT_SCAN: { - auto dn = static_cast<const DistinctNode*>(node); - statsOut->indexesUsed.insert(dn->index.identifier.catalogName); - break; - } - case STAGE_GEO_NEAR_2D: { - auto geo2d = static_cast<const GeoNear2DNode*>(node); - statsOut->indexesUsed.insert(geo2d->index.identifier.catalogName); - break; - } - case STAGE_GEO_NEAR_2DSPHERE: { - auto geo2dsphere = static_cast<const GeoNear2DSphereNode*>(node); - statsOut->indexesUsed.insert(geo2dsphere->index.identifier.catalogName); - break; - } - case STAGE_IXSCAN: { - auto ixn = static_cast<const IndexScanNode*>(node); - statsOut->indexesUsed.insert(ixn->index.identifier.catalogName); - break; - } - case STAGE_TEXT_MATCH: { - auto tn = static_cast<const TextMatchNode*>(node); - statsOut->indexesUsed.insert(tn->index.identifier.catalogName); - break; - } - case STAGE_COLLSCAN: { - statsOut->collectionScans++; - auto csn = static_cast<const CollectionScanNode*>(node); - if (!csn->tailable) { - statsOut->collectionScansNonTailable++; - } - } - default: - break; - } - - for (auto&& child : node->children) { - queue.push(child); - } - } + // Use the pre-computed summary stats instead of traversing the QuerySolution tree. + const auto& indexesUsed = _debugInfo->indexesUsed; + statsOut->indexesUsed.insert(indexesUsed.begin(), indexesUsed.end()); + statsOut->collectionScans += _debugInfo->collectionScans; + statsOut->collectionScansNonTailable += _debugInfo->collectionScansNonTailable; } PlanExplainer::PlanStatsDetails PlanExplainerSBE::getWinningPlanStats( diff --git a/src/mongo/db/query/plan_explainer_sbe.h b/src/mongo/db/query/plan_explainer_sbe.h index bff25e5db60..070ed479647 100644 --- a/src/mongo/db/query/plan_explainer_sbe.h +++ b/src/mongo/db/query/plan_explainer_sbe.h @@ -30,6 +30,7 @@ #pragma once #include "mongo/db/exec/sbe/stages/stages.h" +#include "mongo/db/query/plan_cache_debug_info.h" #include "mongo/db/query/plan_explainer.h" #include "mongo/db/query/query_solution.h" #include "mongo/db/query/sbe_plan_ranker.h" @@ -44,13 +45,17 @@ public: const stage_builder::PlanStageData* data, const QuerySolution* solution, std::vector<sbe::plan_ranker::CandidatePlan> rejectedCandidates, - bool isMultiPlan) + bool isMultiPlan, + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> debugInfo) : PlanExplainer{solution}, _root{root}, _rootData{data}, _solution{solution}, _rejectedCandidates{std::move(rejectedCandidates)}, - _isMultiPlan{isMultiPlan} {} + _isMultiPlan{isMultiPlan}, + _debugInfo{std::move(debugInfo)} { + tassert(5968203, "_debugInfo should not be null", _debugInfo); + } bool isMultiPlan() const final { return _isMultiPlan; @@ -83,5 +88,7 @@ private: const std::vector<sbe::plan_ranker::CandidatePlan> _rejectedCandidates; const bool _isMultiPlan{false}; + // Pre-computed debugging info so we don't necessarily have to collect them from QuerySolution. + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> _debugInfo; }; } // namespace mongo diff --git a/src/mongo/db/query/query_solution.cpp b/src/mongo/db/query/query_solution.cpp index 0f9f633fafa..a0186922e25 100644 --- a/src/mongo/db/query/query_solution.cpp +++ b/src/mongo/db/query/query_solution.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include <queue> #include <vector> #include "mongo/db/query/query_solution.h" @@ -39,6 +40,7 @@ #include "mongo/bson/simple_bsonelement_comparator.h" #include "mongo/db/field_ref.h" #include "mongo/db/index_names.h" +#include "mongo/db/keypattern.h" #include "mongo/db/matcher/expression_geo.h" #include "mongo/db/query/collation/collation_index_key.h" #include "mongo/db/query/index_bounds_builder.h" @@ -150,6 +152,77 @@ bool QuerySolutionNode::hasNode(StageType type) const { return false; } +std::string QuerySolution::summaryString() const { + tassert(5968205, "QuerySolutionNode cannot be null in this QuerySolution", _root); + + StringBuilder sb; + bool seenLeaf = false; + std::queue<const QuerySolutionNode*> queue; + queue.push(_root.get()); + + while (!queue.empty()) { + auto node = queue.front(); + queue.pop(); + + if (node->children.empty()) { + if (seenLeaf) { + sb << ", "; + } else { + seenLeaf = true; + } + + sb << stageTypeToString(node->getType()); + + switch (node->getType()) { + case STAGE_COUNT_SCAN: { + auto csn = static_cast<const CountScanNode*>(node); + const KeyPattern keyPattern{csn->index.keyPattern}; + sb << " " << keyPattern; + break; + } + case STAGE_DISTINCT_SCAN: { + auto dn = static_cast<const DistinctNode*>(node); + const KeyPattern keyPattern{dn->index.keyPattern}; + sb << " " << keyPattern; + break; + } + case STAGE_GEO_NEAR_2D: { + auto geo2d = static_cast<const GeoNear2DNode*>(node); + const KeyPattern keyPattern{geo2d->index.keyPattern}; + sb << " " << keyPattern; + break; + } + case STAGE_GEO_NEAR_2DSPHERE: { + auto geo2dsphere = static_cast<const GeoNear2DSphereNode*>(node); + const KeyPattern keyPattern{geo2dsphere->index.keyPattern}; + sb << " " << keyPattern; + break; + } + case STAGE_IXSCAN: { + auto ixn = static_cast<const IndexScanNode*>(node); + const KeyPattern keyPattern{ixn->index.keyPattern}; + sb << " " << keyPattern; + break; + } + case STAGE_TEXT_MATCH: { + auto tn = static_cast<const TextMatchNode*>(node); + const KeyPattern keyPattern{tn->indexPrefix}; + sb << " " << keyPattern; + break; + } + default: + break; + } + } + + for (auto&& child : node->children) { + queue.push(child); + } + } + + return sb.str(); +} + void QuerySolution::assignNodeIds(QsnIdGenerator& idGenerator, QuerySolutionNode& node) { for (auto&& child : node.children) { assignNodeIds(idGenerator, *child); diff --git a/src/mongo/db/query/query_solution.h b/src/mongo/db/query/query_solution.h index af99bce6c14..c15f5bb9c53 100644 --- a/src/mongo/db/query/query_solution.h +++ b/src/mongo/db/query/query_solution.h @@ -355,6 +355,8 @@ public: return ss; } + std::string summaryString() const; + const QuerySolutionNode* root() const { return _root.get(); } diff --git a/src/mongo/db/query/sbe_cached_solution_planner.cpp b/src/mongo/db/query/sbe_cached_solution_planner.cpp index 1c73d6886bd..f9b7ba125ea 100644 --- a/src/mongo/db/query/sbe_cached_solution_planner.cpp +++ b/src/mongo/db/query/sbe_cached_solution_planner.cpp @@ -46,8 +46,6 @@ namespace mongo::sbe { CandidatePlans CachedSolutionPlanner::plan( std::vector<std::unique_ptr<QuerySolution>> solutions, std::vector<std::pair<std::unique_ptr<PlanStage>, stage_builder::PlanStageData>> roots) { - invariant(solutions.size() == 1); - invariant(solutions.size() == roots.size()); // If the cached plan is accepted we'd like to keep the results from the trials even if there // are parts of agg pipelines being lowered into SBE, so we run the trial with the extended @@ -72,7 +70,14 @@ CandidatePlans CachedSolutionPlanner::plan( std::move(roots[0].second), maxReadsBeforeReplan); auto explainer = plan_explainer_factory::make( - candidate.root.get(), &candidate.data, candidate.solution.get()); + candidate.root.get(), + &candidate.data, + candidate.solution.get(), + {}, /* rejectedCandidates */ + false, /* isMultiPlan */ + candidate.data.debugInfo + ? std::make_unique<plan_cache_debug_info::DebugInfoSBE>(*candidate.data.debugInfo) + : nullptr); if (!candidate.status.isOK()) { // On failure, fall back to replanning the whole query. We neither evict the existing cache @@ -151,6 +156,9 @@ plan_ranker::CandidatePlan CachedSolutionPlanner::collectExecutionStatsForCached std::move(onMetricReached), maxNumResults, maxTrialPeriodNumReads); candidate.root->attachToTrialRunTracker(tracker.get()); executeCandidateTrial(&candidate, maxNumResults); + + candidate.data.debugInfo = std::move(_debugInfo); + return candidate; } diff --git a/src/mongo/db/query/sbe_cached_solution_planner.h b/src/mongo/db/query/sbe_cached_solution_planner.h index a88b844b1c2..ebef51e662b 100644 --- a/src/mongo/db/query/sbe_cached_solution_planner.h +++ b/src/mongo/db/query/sbe_cached_solution_planner.h @@ -49,10 +49,12 @@ public: const CanonicalQuery& cq, const QueryPlannerParams& queryParams, size_t decisionReads, - PlanYieldPolicySBE* yieldPolicy) + PlanYieldPolicySBE* yieldPolicy, + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> debugInfo) : BaseRuntimePlanner{opCtx, collection, cq, yieldPolicy}, _queryParams{queryParams}, - _decisionReads{decisionReads} {} + _decisionReads{decisionReads}, + _debugInfo{std::move(debugInfo)} {} CandidatePlans plan( std::vector<std::unique_ptr<QuerySolution>> solutions, @@ -103,5 +105,8 @@ private: // The number of physical reads taken to decide on a winning plan when the plan was first // cached. const size_t _decisionReads; + + // Stores plan cache entry information used as debug information or for "explain" purpose. + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> _debugInfo; }; } // namespace mongo::sbe diff --git a/src/mongo/db/query/sbe_plan_cache.h b/src/mongo/db/query/sbe_plan_cache.h index 58507412de7..8ee625f4ba3 100644 --- a/src/mongo/db/query/sbe_plan_cache.h +++ b/src/mongo/db/query/sbe_plan_cache.h @@ -80,6 +80,10 @@ public: return hash; } + const std::string& toString() const { + return _info.toString(); + } + private: const PlanCacheKeyInfo _info; const UUID _collectionUuid; @@ -106,7 +110,20 @@ struct PlanCachePartitioner { */ struct CachedSbePlan { CachedSbePlan(std::unique_ptr<sbe::PlanStage> root, stage_builder::PlanStageData data) - : root(std::move(root)), planStageData(std::move(data)) {} + : root(std::move(root)), planStageData(std::move(data)) { + tassert(5968206, "The RuntimeEnvironment should not be null", planStageData.env); + // TODO SERVER-61737: Once the RuntimeEnvironment is deep-copied, there's no need to copy + // collator. + // + // Always make "collator" owned before caching the plan. Because the cached plan should + // outlive collator's original owner. + auto collatorSlot = planStageData.env->getSlotIfExists("collator"_sd); + if (collatorSlot) { + auto collatorCopy = planStageData.env->getAccessor(*collatorSlot)->copyOrMoveValue(); + planStageData.env->resetSlot( + *collatorSlot, collatorCopy.first, collatorCopy.second, true); + } + } std::unique_ptr<CachedSbePlan> clone() const { return std::make_unique<CachedSbePlan>(root->clone(), planStageData); @@ -120,7 +137,7 @@ struct CachedSbePlan { stage_builder::PlanStageData planStageData; }; -using PlanCacheEntry = PlanCacheEntryBase<CachedSbePlan>; +using PlanCacheEntry = PlanCacheEntryBase<CachedSbePlan, plan_cache_debug_info::DebugInfoSBE>; struct BudgetEstimator { size_t operator()(const PlanCacheEntry& entry) { @@ -131,6 +148,7 @@ struct BudgetEstimator { using PlanCache = PlanCacheBase<PlanCacheKey, CachedSbePlan, BudgetEstimator, + plan_cache_debug_info::DebugInfoSBE, PlanCachePartitioner, PlanCacheKeyHasher>; diff --git a/src/mongo/db/query/sbe_stage_builder.h b/src/mongo/db/query/sbe_stage_builder.h index 95eaff653ac..e92a8d4e30e 100644 --- a/src/mongo/db/query/sbe_stage_builder.h +++ b/src/mongo/db/query/sbe_stage_builder.h @@ -267,6 +267,10 @@ struct PlanStageData { // metrics, the stats are cached in here. std::unique_ptr<sbe::PlanStageStats> savedStatsOnEarlyExit{nullptr}; + // Stores plan cache entry information used as debug information or for "explain" purpose. + // Note that 'debugInfo' is present only if this PlanStageData is recovered from the plan cache. + std::unique_ptr<plan_cache_debug_info::DebugInfoSBE> debugInfo; + private: // This copy function copies data from 'other' but will not create a copy of its // RuntimeEnvironment and CompileCtx. @@ -282,6 +286,11 @@ private: } else { savedStatsOnEarlyExit.reset(); } + if (other.debugInfo) { + debugInfo = std::make_unique<plan_cache_debug_info::DebugInfoSBE>(*other.debugInfo); + } else { + debugInfo.reset(); + } } }; |