SERVER-47640 $lookup should increment the serverStatus metrics.queryExecutor counters

7 files changed, 118 insertions, 18 deletions

diff --git a/src/mongo/db/exec/plan_stats.h b/src/mongo/db/exec/plan_stats.h
index 8e1e7df62b2..b411b36647f 100644
--- a/src/mongo/db/exec/plan_stats.h
+++ b/src/mongo/db/exec/plan_stats.h
@@ -34,8 +34,6 @@
 #include <string>
 #include <vector>
 
-#include "mongo/db/exec/document_value/document.h"
-#include "mongo/db/exec/document_value/value.h"
 #include "mongo/db/index/multikey_paths.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/query/plan_summary_stats.h"
@@ -860,4 +858,39 @@ struct GroupStats : public SpecificStats {
     bool usedDisk = false;
 };
 
+struct DocumentSourceCursorStats : public SpecificStats {
+    SpecificStats* clone() const final {
+        return new DocumentSourceCursorStats(*this);
+    }
+
+    uint64_t estimateObjectSizeInBytes() const {
+        return sizeof(*this) +
+            (planSummaryStats.estimateObjectSizeInBytes() - sizeof(planSummaryStats));
+    }
+
+    void accumulate(PlanSummaryStats& summary) const final {
+        summary.accumulate(planSummaryStats);
+    }
+
+    PlanSummaryStats planSummaryStats;
+};
+
+struct DocumentSourceLookupStats : public SpecificStats {
+    SpecificStats* clone() const final {
+        return new DocumentSourceLookupStats(*this);
+    }
+
+    uint64_t estimateObjectSizeInBytes() const {
+        return sizeof(*this) +
+            (planSummaryStats.estimateObjectSizeInBytes() - sizeof(planSummaryStats));
+    }
+
+    void accumulate(PlanSummaryStats& summary) const final {
+        summary.accumulate(planSummaryStats);
+    }
+
+    // Tracks the summary stats in aggregate across all executions of the subpipeline.
+    PlanSummaryStats planSummaryStats;
+};
+
 }  // namespace mongo
diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp
index 341320c9a64..71a019901e5 100644
--- a/src/mongo/db/pipeline/document_source_cursor.cpp
+++ b/src/mongo/db/pipeline/document_source_cursor.cpp
@@ -195,7 +195,7 @@ void DocumentSourceCursor::_updateOplogTimestamp() {
 
 void DocumentSourceCursor::recordPlanSummaryStats() {
     invariant(_exec);
-    _exec->getPlanExplainer().getSummaryStats(&_planSummaryStats);
+    _exec->getPlanExplainer().getSummaryStats(&_stats.planSummaryStats);
 }
 
 Value DocumentSourceCursor::serialize(boost::optional<ExplainOptions::Verbosity> verbosity) const {
@@ -314,7 +314,7 @@ DocumentSourceCursor::DocumentSourceCursor(
 
     if (collection) {
         CollectionQueryInfo::get(collection)
-            .notifyOfQuery(pExpCtx->opCtx, collection, _planSummaryStats);
+            .notifyOfQuery(pExpCtx->opCtx, collection, _stats.planSummaryStats);
     }
 }
 
diff --git a/src/mongo/db/pipeline/document_source_cursor.h b/src/mongo/db/pipeline/document_source_cursor.h
index 360078f90a9..6df87f7c706 100644
--- a/src/mongo/db/pipeline/document_source_cursor.h
+++ b/src/mongo/db/pipeline/document_source_cursor.h
@@ -104,11 +104,15 @@ public:
     }
 
     const PlanSummaryStats& getPlanSummaryStats() const {
-        return _planSummaryStats;
+        return _stats.planSummaryStats;
     }
 
     bool usedDisk() final {
-        return _planSummaryStats.usedDisk;
+        return _stats.planSummaryStats.usedDisk;
+    }
+
+    const SpecificStats* getSpecificStats() const final {
+        return &_stats;
     }
 
 protected:
@@ -229,7 +233,6 @@ private:
     Status _execStatus = Status::OK();
 
     std::string _planSummary;
-    PlanSummaryStats _planSummaryStats;
 
     // Used only for explain() queries. Stores the stats of the winning plan when a plan was
     // selected by the multi-planner. When the query is executed (with exec->executePlan()), it will
@@ -242,6 +245,9 @@ private:
     // If we are tailing the oplog and tracking the latest observed oplog time, this is the latest
     // timestamp seen in the collection. Otherwise, this is a null timestamp.
     Timestamp _latestOplogTimestamp;
+
+    // Specific stats for $cursor stage.
+    DocumentSourceCursorStats _stats;
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp
index 8e166898e0c..099109366fc 100644
--- a/src/mongo/db/pipeline/document_source_lookup.cpp
+++ b/src/mongo/db/pipeline/document_source_lookup.cpp
@@ -328,8 +328,8 @@ DocumentSource::GetNextResult DocumentSourceLookUp::doGetNext() {
         objsize = safeSum;
         results.emplace_back(std::move(*result));
     }
-    _usedDisk = _usedDisk || pipeline->usedDisk();
 
+    recordPlanSummaryStats(*pipeline);
     MutableDocument output(std::move(inputDoc));
     output.setNestedField(_as, Value(std::move(results)));
     return output.freeze();
@@ -529,13 +529,15 @@ Pipeline::SourceContainer::iterator DocumentSourceLookUp::doOptimizeAt(
 
 bool DocumentSourceLookUp::usedDisk() {
     if (_pipeline)
-        _usedDisk = _usedDisk || _pipeline->usedDisk();
-    return _usedDisk;
+        _stats.planSummaryStats.usedDisk =
+            _stats.planSummaryStats.usedDisk || _pipeline->usedDisk();
+
+    return _stats.planSummaryStats.usedDisk;
 }
 
 void DocumentSourceLookUp::doDispose() {
     if (_pipeline) {
-        _usedDisk = _usedDisk || _pipeline->usedDisk();
+        recordPlanSummaryStats(*_pipeline);
         _pipeline->dispose(pExpCtx->opCtx);
         _pipeline.reset();
     }
@@ -648,7 +650,7 @@ DocumentSource::GetNextResult DocumentSourceLookUp::unwindResult() {
         }
 
         if (_pipeline) {
-            _usedDisk = _usedDisk || _pipeline->usedDisk();
+            recordPlanSummaryStats(*_pipeline);
             _pipeline->dispose(pExpCtx->opCtx);
         }
 
@@ -707,6 +709,14 @@ void DocumentSourceLookUp::initializeResolvedIntrospectionPipeline() {
         Pipeline::parse(_resolvedPipeline, _fromExpCtx, lookupPipeValidator);
 }
 
+void DocumentSourceLookUp::recordPlanSummaryStats(const Pipeline& pipeline) {
+    for (auto&& source : pipeline.getSources()) {
+        if (auto specificStats = source->getSpecificStats()) {
+            specificStats->accumulate(_stats.planSummaryStats);
+        }
+    }
+}
+
 void DocumentSourceLookUp::serializeToArray(
     std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const {
     Document doc;
diff --git a/src/mongo/db/pipeline/document_source_lookup.h b/src/mongo/db/pipeline/document_source_lookup.h
index 63e17f643f5..0b1e1457e7c 100644
--- a/src/mongo/db/pipeline/document_source_lookup.h
+++ b/src/mongo/db/pipeline/document_source_lookup.h
@@ -115,6 +115,10 @@ public:
 
     bool usedDisk() final;
 
+    const SpecificStats* getSpecificStats() const final {
+        return &_stats;
+    }
+
     static boost::intrusive_ptr<DocumentSource> createFromBson(
         BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx);
 
@@ -267,7 +271,13 @@ private:
         _cache.emplace(maxCacheSizeBytes);
     }
 
-    bool _usedDisk = false;
+    /**
+     * Method to accumulate the plan summary stats from all stages of the pipeline.
+     */
+    void recordPlanSummaryStats(const Pipeline& pipeline);
+
+    DocumentSourceLookupStats _stats;
+
     NamespaceString _fromNs;
     NamespaceString _resolvedNs;
     FieldPath _as;
diff --git a/src/mongo/db/pipeline/plan_explainer_pipeline.cpp b/src/mongo/db/pipeline/plan_explainer_pipeline.cpp
index 550e8b7135c..64759c4ca3c 100644
--- a/src/mongo/db/pipeline/plan_explainer_pipeline.cpp
+++ b/src/mongo/db/pipeline/plan_explainer_pipeline.cpp
@@ -32,6 +32,7 @@
 #include "mongo/db/pipeline/plan_explainer_pipeline.h"
 
 #include "mongo/db/pipeline/document_source_cursor.h"
+#include "mongo/db/pipeline/document_source_lookup.h"
 #include "mongo/db/pipeline/document_source_sort.h"
 #include "mongo/db/pipeline/plan_executor_pipeline.h"
 #include "mongo/db/query/explain.h"
@@ -55,12 +56,18 @@ void PlanExplainerPipeline::getSummaryStats(PlanSummaryStats* statsOut) const {
     }
 
     for (auto&& source : _pipeline->getSources()) {
-        if (dynamic_cast<DocumentSourceSort*>(source.get()))
-            statsOut->hasSortStage = true;
-
         statsOut->usedDisk = statsOut->usedDisk || source->usedDisk();
-        if (statsOut->usedDisk && statsOut->hasSortStage)
-            break;
+
+        if (dynamic_cast<DocumentSourceSort*>(source.get())) {
+            statsOut->hasSortStage = true;
+        } else if (auto docSourceLookUp = dynamic_cast<DocumentSourceLookUp*>(source.get())) {
+            auto specificStats = docSourceLookUp->getSpecificStats();
+            invariant(specificStats);
+            auto lookupSpecificStats =
+                dynamic_cast<const DocumentSourceLookupStats*>(specificStats);
+            invariant(lookupSpecificStats);
+            statsOut->accumulate(lookupSpecificStats->planSummaryStats);
+        }
     }
 
     if (_nReturned) {
diff --git a/src/mongo/db/query/plan_summary_stats.h b/src/mongo/db/query/plan_summary_stats.h
index 8d0603baee7..58677ab20d2 100644
--- a/src/mongo/db/query/plan_summary_stats.h
+++ b/src/mongo/db/query/plan_summary_stats.h
@@ -29,6 +29,7 @@
 
 #pragma once
 
+#include "mongo/util/container_size_helper.h"
 #include <string>
 
 namespace mongo {
@@ -38,6 +39,39 @@ namespace mongo {
  * other non-explain debug mechanisms may want to collect.
  */
 struct PlanSummaryStats {
+    /**
+     * Helper method to accumulate the plan summary stats from the input source.
+     */
+    void accumulate(const PlanSummaryStats& statsIn) {
+        // Attributes replanReason and fromMultiPlanner have been intentionally skipped as they
+        // always describe the left-hand side (or "local") collection.
+        // Consider $lookup case. $lookup runtime plan selection may happen against the foreign
+        // collection an arbitrary number of times. A single value of 'replanReason' and
+        // 'fromMultiPlanner' can't really report correctly on the behavior of arbitrarily many
+        // occurrences of runtime planning for a single query.
+
+        nReturned += statsIn.nReturned;
+        totalKeysExamined += statsIn.totalKeysExamined;
+        totalDocsExamined += statsIn.totalDocsExamined;
+        collectionScans += statsIn.collectionScans;
+        collectionScansNonTailable += statsIn.collectionScansNonTailable;
+        hasSortStage |= statsIn.hasSortStage;
+        usedDisk |= statsIn.usedDisk;
+        planFailed |= statsIn.planFailed;
+        indexesUsed.insert(statsIn.indexesUsed.begin(), statsIn.indexesUsed.end());
+    }
+
+    uint64_t estimateObjectSizeInBytes() const {
+        auto strSize = [](const std::string& str) {
+            return str.capacity() * sizeof(std::string::value_type);
+        };
+
+        return sizeof(*this) +
+            container_size_helper::estimateObjectSizeInBytes(
+                   indexesUsed, strSize, false /* includeShallowSize */) +
+            (replanReason ? strSize(*replanReason) : 0);
+    }
+
     // The number of results returned by the plan.
     size_t nReturned = 0U;