From e201049a0ddd15704c348863e75d7f1473ae0beb Mon Sep 17 00:00:00 2001 From: Bikash Chandra Date: Thu, 26 Aug 2021 18:51:06 +0530 Subject: SERVER-53771 Facet summary stats added --- jstests/aggregation/sources/facet/facet_stats.js | 56 ++++++++++++++++++++++ src/mongo/db/exec/plan_stats.h | 18 +++++++ src/mongo/db/pipeline/document_source.cpp | 9 ++++ src/mongo/db/pipeline/document_source.h | 7 +++ src/mongo/db/pipeline/document_source_facet.cpp | 9 ++-- src/mongo/db/pipeline/document_source_facet.h | 5 ++ src/mongo/db/pipeline/document_source_lookup.cpp | 14 ++---- src/mongo/db/pipeline/document_source_lookup.h | 5 -- .../db/pipeline/document_source_union_with.cpp | 12 +---- src/mongo/db/pipeline/document_source_union_with.h | 2 - src/mongo/db/pipeline/plan_explainer_pipeline.cpp | 4 ++ 11 files changed, 110 insertions(+), 31 deletions(-) create mode 100644 jstests/aggregation/sources/facet/facet_stats.js diff --git a/jstests/aggregation/sources/facet/facet_stats.js b/jstests/aggregation/sources/facet/facet_stats.js new file mode 100644 index 00000000000..fdbab25975e --- /dev/null +++ b/jstests/aggregation/sources/facet/facet_stats.js @@ -0,0 +1,56 @@ +// Test that the $facet stage reports the correct stats in serverStatus().metrics.queryExecutor. +// @tags: [ +// # Should not run on sharded suites due to use of serverStatus() +// assumes_unsharded_collection, +// assumes_no_implicit_collection_creation_after_drop, +// do_not_wrap_aggregations_in_facets, +// assumes_read_preference_unchanged, +// assumes_read_concern_unchanged, +// assumes_against_mongod_not_mongos +// ] + +(function() { +"use strict"; + +const testDB = db.getSiblingDB("facet_stats"); +const local = testDB.facetLookupLocal; +const foreign = testDB.facetLookupForeign; +testDB.dropDatabase(); + +let runFacetPipeline = function() { + const lookup = { + $lookup: { + from: foreign.getName(), + let: {id1: "$_id"}, + pipeline: [{$match: {$expr: {$eq: ["$$id1", "$foreignKey"]}}}], + as: "joined" + } + }; + + return local.aggregate([{$facet: {nested: [lookup]}}]).itcount(); +}; + +assert.commandWorked(local.insert({_id: 1, score: 100})); +assert.commandWorked(local.insert({_id: 2, score: 200})); +assert.commandWorked(local.insert({_id: 3, score: 200})); + +assert.commandWorked(foreign.insert({_id: 0, foreignKey: 1})); +assert.commandWorked(foreign.insert({_id: 1, foreignKey: 2})); +assert.commandWorked(foreign.insert({_id: 2, foreignKey: 3})); + +let queryExecutor = testDB.serverStatus().metrics.queryExecutor; +let curScannedObjects = queryExecutor.scannedObjects; +let curScannedKeys = queryExecutor.scanned; + +assert.eq(1, runFacetPipeline()); + +queryExecutor = testDB.serverStatus().metrics.queryExecutor; +curScannedObjects = queryExecutor.scannedObjects - curScannedObjects; +curScannedKeys = queryExecutor.scanned - curScannedKeys; +// For each document on the local side, this query has to perform a scan of the foreign side. +// Therefore, the total number of documents examined is +// cardinality(local) + cardinality(local) * cardinality(foreign) = 3 + 3 * 3 = 12. +assert.eq(12, curScannedObjects); +// $facet sub-pipelines cannot make use of indexes. Hence scanned keys should be 0. +assert.eq(0, curScannedKeys); +})(); diff --git a/src/mongo/db/exec/plan_stats.h b/src/mongo/db/exec/plan_stats.h index fd662242b72..0c2aac4425a 100644 --- a/src/mongo/db/exec/plan_stats.h +++ b/src/mongo/db/exec/plan_stats.h @@ -864,6 +864,24 @@ struct UnionWithStats final : public SpecificStats { PlanSummaryStats planSummaryStats; }; +struct DocumentSourceFacetStats : public SpecificStats { + std::unique_ptr clone() const final { + return std::make_unique(*this); + } + + uint64_t estimateObjectSizeInBytes() const { + return sizeof(*this) + + (planSummaryStats.estimateObjectSizeInBytes() - sizeof(planSummaryStats)); + } + + void accumulate(PlanSummaryStats& summary) const final { + summary.accumulate(planSummaryStats); + } + + // Tracks the cumulative summary stats across all facets. + PlanSummaryStats planSummaryStats; +}; + struct UnpackTimeseriesBucketStats final : public SpecificStats { std::unique_ptr clone() const final { return std::make_unique(*this); diff --git a/src/mongo/db/pipeline/document_source.cpp b/src/mongo/db/pipeline/document_source.cpp index b4cf3a59362..f86e99022da 100644 --- a/src/mongo/db/pipeline/document_source.cpp +++ b/src/mongo/db/pipeline/document_source.cpp @@ -74,6 +74,15 @@ struct ParserRegistration { static StringMap parserMap; } // namespace +void accumulatePipelinePlanSummaryStats(const Pipeline& pipeline, + PlanSummaryStats& planSummaryStats) { + for (auto&& source : pipeline.getSources()) { + if (auto specificStats = source->getSpecificStats()) { + specificStats->accumulate(planSummaryStats); + } + } +} + void DocumentSource::registerParser( string name, Parser parser, diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index b75fc71ee15..cd86955a142 100644 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -709,4 +709,11 @@ private: boost::optional explain = boost::none) const = 0; }; +/** + * Method to accumulate the plan summary stats from all stages of the pipeline into the given + * `planSummaryStats` object. + */ +void accumulatePipelinePlanSummaryStats(const Pipeline& pipeline, + PlanSummaryStats& planSummaryStats); + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_facet.cpp b/src/mongo/db/pipeline/document_source_facet.cpp index 4d8da6de141..60f1394c726 100644 --- a/src/mongo/db/pipeline/document_source_facet.cpp +++ b/src/mongo/db/pipeline/document_source_facet.cpp @@ -172,6 +172,7 @@ DocumentSource::GetNextResult DocumentSourceFacet::doGetNext() { results[facetId].emplace_back(next.releaseDocument()); } allPipelinesEOF = allPipelinesEOF && next.isEOF(); + accumulatePipelinePlanSummaryStats(*pipeline, _stats.planSummaryStats); } } @@ -264,10 +265,12 @@ StageConstraints DocumentSourceFacet::constraints(Pipeline::SplitState) const { bool DocumentSourceFacet::usedDisk() { for (auto&& facet : _facets) { - if (facet.pipeline->usedDisk()) - return true; + if (facet.pipeline->usedDisk()) { + _stats.planSummaryStats.usedDisk = true; + break; + } } - return false; + return _stats.planSummaryStats.usedDisk; } DepsTracker::State DocumentSourceFacet::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_facet.h b/src/mongo/db/pipeline/document_source_facet.h index f5c18e4352d..3cc29294b5a 100644 --- a/src/mongo/db/pipeline/document_source_facet.h +++ b/src/mongo/db/pipeline/document_source_facet.h @@ -141,6 +141,9 @@ public: void reattachToOperationContext(OperationContext* opCtx) final; StageConstraints constraints(Pipeline::SplitState pipeState) const final; bool usedDisk() final; + const SpecificStats* getSpecificStats() const final { + return &_stats; + } protected: /** @@ -163,5 +166,7 @@ private: const size_t _maxOutputDocSizeBytes; bool _done = false; + + DocumentSourceFacetStats _stats; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp index eefa1e13626..4edce8a721b 100644 --- a/src/mongo/db/pipeline/document_source_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_lookup.cpp @@ -480,7 +480,7 @@ DocumentSource::GetNextResult DocumentSourceLookUp::doGetNext() { results.emplace_back(std::move(*result)); } - recordPlanSummaryStats(*pipeline); + accumulatePipelinePlanSummaryStats(*pipeline, _stats.planSummaryStats); MutableDocument output(std::move(inputDoc)); output.setNestedField(_as, Value(std::move(results))); return output.freeze(); @@ -806,7 +806,7 @@ bool DocumentSourceLookUp::usedDisk() { void DocumentSourceLookUp::doDispose() { if (_pipeline) { - recordPlanSummaryStats(*_pipeline); + accumulatePipelinePlanSummaryStats(*_pipeline, _stats.planSummaryStats); _pipeline->dispose(pExpCtx->opCtx); _pipeline.reset(); } @@ -922,7 +922,7 @@ DocumentSource::GetNextResult DocumentSourceLookUp::unwindResult() { } if (_pipeline) { - recordPlanSummaryStats(*_pipeline); + accumulatePipelinePlanSummaryStats(*_pipeline, _stats.planSummaryStats); _pipeline->dispose(pExpCtx->opCtx); } @@ -983,14 +983,6 @@ void DocumentSourceLookUp::initializeResolvedIntrospectionPipeline() { _fromExpCtx->stopExpressionCounters(); } -void DocumentSourceLookUp::recordPlanSummaryStats(const Pipeline& pipeline) { - for (auto&& source : pipeline.getSources()) { - if (auto specificStats = source->getSpecificStats()) { - specificStats->accumulate(_stats.planSummaryStats); - } - } -} - void DocumentSourceLookUp::appendSpecificExecStats(MutableDocument& doc) const { const PlanSummaryStats& stats = _stats.planSummaryStats; doc["totalDocsExamined"] = Value(static_cast(stats.totalDocsExamined)); diff --git a/src/mongo/db/pipeline/document_source_lookup.h b/src/mongo/db/pipeline/document_source_lookup.h index cf22eca3073..74b71070fbe 100644 --- a/src/mongo/db/pipeline/document_source_lookup.h +++ b/src/mongo/db/pipeline/document_source_lookup.h @@ -307,11 +307,6 @@ private: _cache.emplace(maxCacheSizeBytes); } - /** - * Method to accumulate the plan summary stats from all stages of the pipeline. - */ - void recordPlanSummaryStats(const Pipeline& pipeline); - /** * Method to add a DocumentSourceSequentialDocumentCache stage and optimize the pipeline to * move the cache to its final position. diff --git a/src/mongo/db/pipeline/document_source_union_with.cpp b/src/mongo/db/pipeline/document_source_union_with.cpp index 7d1e4ea5568..2ddb448b641 100644 --- a/src/mongo/db/pipeline/document_source_union_with.cpp +++ b/src/mongo/db/pipeline/document_source_union_with.cpp @@ -252,7 +252,7 @@ DocumentSource::GetNextResult DocumentSourceUnionWith::doGetNext() { return std::move(*res); // Record the plan summary stats after $unionWith operation is done. - recordPlanSummaryStats(*_pipeline); + accumulatePipelinePlanSummaryStats(*_pipeline, _stats.planSummaryStats); _executionState = ExecutionProgress::kFinished; return GetNextResult::makeEOF(); @@ -318,7 +318,7 @@ void DocumentSourceUnionWith::doDispose() { _pipeline.get_deleter().dismissDisposal(); _stats.planSummaryStats.usedDisk = _stats.planSummaryStats.usedDisk || _pipeline->usedDisk(); - recordPlanSummaryStats(*_pipeline); + accumulatePipelinePlanSummaryStats(*_pipeline, _stats.planSummaryStats); if (!_pipeline->getContext()->explain) { _pipeline->dispose(pExpCtx->opCtx); @@ -412,12 +412,4 @@ void DocumentSourceUnionWith::addInvolvedCollections( collectionNames->merge(_pipeline->getInvolvedCollections()); } -void DocumentSourceUnionWith::recordPlanSummaryStats(const Pipeline& pipeline) { - for (auto&& source : pipeline.getSources()) { - if (auto specificStats = source->getSpecificStats()) { - specificStats->accumulate(_stats.planSummaryStats); - } - } -} - } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_union_with.h b/src/mongo/db/pipeline/document_source_union_with.h index 9219e08c48c..1d9068a9f4c 100644 --- a/src/mongo/db/pipeline/document_source_union_with.h +++ b/src/mongo/db/pipeline/document_source_union_with.h @@ -170,8 +170,6 @@ private: void addViewDefinition(NamespaceString nss, std::vector viewPipeline); - void recordPlanSummaryStats(const Pipeline& pipeline); - void logStartingSubPipeline(const std::vector& serializedPipeline); void logShardedViewFound( const ExceptionFor& e); diff --git a/src/mongo/db/pipeline/plan_explainer_pipeline.cpp b/src/mongo/db/pipeline/plan_explainer_pipeline.cpp index 072643fd7e6..cd138db0466 100644 --- a/src/mongo/db/pipeline/plan_explainer_pipeline.cpp +++ b/src/mongo/db/pipeline/plan_explainer_pipeline.cpp @@ -32,6 +32,7 @@ #include "mongo/db/pipeline/plan_explainer_pipeline.h" #include "mongo/db/pipeline/document_source_cursor.h" +#include "mongo/db/pipeline/document_source_facet.h" #include "mongo/db/pipeline/document_source_lookup.h" #include "mongo/db/pipeline/document_source_sort.h" #include "mongo/db/pipeline/document_source_union_with.h" @@ -88,6 +89,9 @@ void PlanExplainerPipeline::getSummaryStats(PlanSummaryStats* statsOut) const { } else if (auto docSourceUnionWith = dynamic_cast(source.get())) { collectPlanSummaryStats(*docSourceUnionWith, statsOut); + } else if (auto docSourceFacet = dynamic_cast(source.get())) { + collectPlanSummaryStats(*docSourceFacet, + statsOut); } } -- cgit v1.2.1