SERVER-64539 Add serverStatus metrics for the SBE $lookup project

7 files changed, 76 insertions, 20 deletions

diff --git a/src/mongo/db/curop.h b/src/mongo/db/curop.h
index b641ae3547f..a2877758fd0 100644
--- a/src/mongo/db/curop.h
+++ b/src/mongo/db/curop.h
@@ -270,6 +270,21 @@ public:
     // involved in the execution tree for this query, false if they were not.
     boost::optional<bool> documentSourceUsed;
 
+    // Tracks whether an aggregation query has a lookup stage regardless of the engine used.
+    bool pipelineUsesLookup{false};
+
+    // Tracks the amount of indexed loop joins in a pushed down lookup stage.
+    int indexedLoopJoin{0};
+
+    // Tracks the amount of nested loop joins in a pushed down lookup stage.
+    int nestedLoopJoin{0};
+
+    // Tracks the amount of hash lookups in a pushed down lookup stage.
+    int hashLookup{0};
+
+    // Tracks the amount of spills by hash lookup in a pushed down lookup stage.
+    int hashLookupSpillToDisk{0};
+
     // Details of any error (whether from an exception or a command returning failure).
     Status errInfo = Status::OK();
 
diff --git a/src/mongo/db/curop_metrics.cpp b/src/mongo/db/curop_metrics.cpp
index 526f779efa2..728c79472de 100644
--- a/src/mongo/db/curop_metrics.cpp
+++ b/src/mongo/db/curop_metrics.cpp
@@ -68,6 +68,8 @@ void recordCurOpMetrics(OperationContext* opCtx) {
     if (auto n = debug.additiveMetrics.writeConflicts.load(); n > 0)
         writeConflictsCounter.increment(n);
 
+    lookupPushdownCounters.incrementLookupCounters(CurOp::get(opCtx)->debug());
+
     queryEngineCounters.incrementQueryEngineCounters(CurOp::get(opCtx));
 }
 
diff --git a/src/mongo/db/exec/sbe/stages/hash_lookup.cpp b/src/mongo/db/exec/sbe/stages/hash_lookup.cpp
index 16e61d68630..8c760ed0f05 100644
--- a/src/mongo/db/exec/sbe/stages/hash_lookup.cpp
+++ b/src/mongo/db/exec/sbe/stages/hash_lookup.cpp
@@ -29,6 +29,7 @@
 
 #include "mongo/platform/basic.h"
 
+#include "mongo/db/curop.h"
 #include "mongo/db/exec/sbe/stages/hash_lookup.h"
 #include "mongo/db/exec/sbe/stages/stage_visitors.h"
 
@@ -312,6 +313,8 @@ void HashLookupStage::spillBufferedValueToDisk(OperationContext* opCtx,
                                                RecordStore* rs,
                                                size_t bufferIdx,
                                                const value::MaterializedRow& val) {
+    CurOp::get(_opCtx)->debug().hashLookupSpillToDisk += 1;
+
     auto rid = getValueRecordId(bufferIdx);
 
     BufBuilder buf;
@@ -496,6 +499,8 @@ void HashLookupStage::spillIndicesToRecordStore(RecordStore* rs,
                                                 value::TypeTags tagKey,
                                                 value::Value valKey,
                                                 const std::vector<size_t>& value) {
+    CurOp::get(_opCtx)->debug().hashLookupSpillToDisk += 1;
+
     auto [owned, tagKeyColl, valKeyColl] = normalizeStringIfCollator(tagKey, valKey);
     _probeKey.reset(0, owned, tagKeyColl, valKeyColl);
 
diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp
index 70dc18d38f1..2132049557a 100644
--- a/src/mongo/db/pipeline/pipeline_d.cpp
+++ b/src/mongo/db/pipeline/pipeline_d.cpp
@@ -40,6 +40,7 @@
 #include "mongo/db/catalog/database.h"
 #include "mongo/db/catalog/index_catalog.h"
 #include "mongo/db/concurrency/d_concurrency.h"
+#include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/exec/cached_plan.h"
 #include "mongo/db/exec/collection_scan.h"
@@ -184,6 +185,7 @@ std::vector<std::unique_ptr<InnerPipelineStageInterface>> extractSbeCompatibleSt
 
         // $lookup pushdown logic.
         if (auto lookupStage = dynamic_cast<DocumentSourceLookUp*>(itr->get())) {
+            CurOp::get(expCtx->opCtx)->debug().pipelineUsesLookup = true;
             if (disallowLookupPushdown) {
                 break;
             }
@@ -223,7 +225,6 @@ std::vector<std::unique_ptr<InnerPipelineStageInterface>> extractSbeCompatibleSt
         // Current stage cannot be pushed down.
         break;
     }
-
     return stagesForPushdown;
 }
 
diff --git a/src/mongo/db/query/sbe_stage_builder_lookup.cpp b/src/mongo/db/query/sbe_stage_builder_lookup.cpp
index a83614792e9..226e3cead83 100644
--- a/src/mongo/db/query/sbe_stage_builder_lookup.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_lookup.cpp
@@ -34,6 +34,7 @@
 
 #include <fmt/format.h>
 
+#include "mongo/db/curop.h"
 #include "mongo/db/exec/sbe/stages/branch.h"
 #include "mongo/db/exec/sbe/stages/hash_agg.h"
 #include "mongo/db/exec/sbe/stages/hash_lookup.h"
@@ -508,6 +509,7 @@ std::pair<SlotId /* matched docs */, std::unique_ptr<sbe::PlanStage>> buildForei
 }
 
 std::pair<SlotId /* matched docs */, std::unique_ptr<sbe::PlanStage>> buildNljLookupStage(
+    StageBuilderState& state,
     std::unique_ptr<sbe::PlanStage> localStage,
     SlotId localRecordSlot,
     const FieldPath& localFieldName,
@@ -516,8 +518,9 @@ std::pair<SlotId /* matched docs */, std::unique_ptr<sbe::PlanStage>> buildNljLo
     StringData foreignFieldName,
     boost::optional<SlotId> collatorSlot,
     const PlanNodeId nodeId,
-    SlotIdGenerator& slotIdGenerator,
-    bool allowDiskUse) {
+    SlotIdGenerator& slotIdGenerator) {
+    CurOp::get(state.opCtx)->debug().nestedLoopJoin += 1;
+
     // Build the outer branch that produces the set of local key values.
     auto [localKeySlot, outerRootStage] = buildKeySet(JoinSide::Local,
                                                       std::move(localStage),
@@ -525,7 +528,7 @@ std::pair<SlotId /* matched docs */, std::unique_ptr<sbe::PlanStage>> buildNljLo
                                                       localFieldName,
                                                       nodeId,
                                                       slotIdGenerator,
-                                                      allowDiskUse);
+                                                      state.allowDiskUse);
 
     // Build the inner branch that will get the foreign key values, compare them to the local key
     // values and accumulate all matching foreign records into an array that is placed into
@@ -537,7 +540,7 @@ std::pair<SlotId /* matched docs */, std::unique_ptr<sbe::PlanStage>> buildNljLo
                                                                     collatorSlot,
                                                                     nodeId,
                                                                     slotIdGenerator,
-                                                                    allowDiskUse);
+                                                                    state.allowDiskUse);
 
     // 'innerRootStage' should not participate in trial run tracking as the number of reads that
     // it performs should not influence planning decisions made for 'outerRootStage'.
@@ -619,6 +622,8 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
     SlotIdGenerator& slotIdGenerator,
     FrameIdGenerator& frameIdGenerator,
     RuntimeEnvironment* env) {
+    CurOp::get(state.opCtx)->debug().indexedLoopJoin += 1;
+
     const auto foreignCollUUID = foreignColl->uuid();
     const auto indexName = index.identifier.catalogName;
     const auto indexDescriptor =
@@ -877,6 +882,7 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
 }
 
 std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildHashJoinLookupStage(
+    StageBuilderState& state,
     std::unique_ptr<sbe::PlanStage> localStage,
     SlotId localRecordSlot,
     const FieldPath& localFieldName,
@@ -885,8 +891,8 @@ std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildHashJoi
     const FieldPath& foreignFieldName,
     boost::optional<SlotId> collatorSlot,
     const PlanNodeId nodeId,
-    SlotIdGenerator& slotIdGenerator,
-    bool allowDiskUse) {
+    SlotIdGenerator& slotIdGenerator) {
+    CurOp::get(state.opCtx)->debug().hashLookup += 1;
 
     // Build the outer branch that produces the set of local key values.
     auto [localKeySlot, outerRootStage] = buildKeySet(JoinSide::Local,
@@ -895,7 +901,7 @@ std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildHashJoi
                                                       localFieldName,
                                                       nodeId,
                                                       slotIdGenerator,
-                                                      allowDiskUse);
+                                                      state.allowDiskUse);
 
     // Build the inner branch that produces the set of foreign key values.
     auto [foreignKeySlot, foreignKeyStage] = buildKeySet(JoinSide::Foreign,
@@ -904,7 +910,7 @@ std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildHashJoi
                                                          foreignFieldName,
                                                          nodeId,
                                                          slotIdGenerator,
-                                                         allowDiskUse);
+                                                         state.allowDiskUse);
 
     // 'foreignKeyStage' should not participate in trial run tracking as the number of
     // reads that it performs should not influence planning decisions for 'outerRootStage'.
@@ -937,6 +943,7 @@ std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildHashJoi
 }
 
 std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildLookupStage(
+    StageBuilderState& state,
     EqLookupNode::LookupStrategy lookupStrategy,
     std::unique_ptr<sbe::PlanStage> localStage,
     SlotId localRecordSlot,
@@ -946,11 +953,11 @@ std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildLookupS
     const FieldPath& foreignFieldName,
     boost::optional<SlotId> collatorSlot,
     const PlanNodeId nodeId,
-    SlotIdGenerator& slotIdGenerator,
-    bool allowDiskUse) {
+    SlotIdGenerator& slotIdGenerator) {
     switch (lookupStrategy) {
         case EqLookupNode::LookupStrategy::kNestedLoopJoin:
-            return buildNljLookupStage(std::move(localStage),
+            return buildNljLookupStage(state,
+                                       std::move(localStage),
                                        localRecordSlot,
                                        localFieldName,
                                        std::move(foreignStage),
@@ -958,10 +965,10 @@ std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildLookupS
                                        foreignFieldName.fullPath(),
                                        collatorSlot,
                                        nodeId,
-                                       slotIdGenerator,
-                                       allowDiskUse);
+                                       slotIdGenerator);
         case EqLookupNode::LookupStrategy::kHashJoin:
-            return buildHashJoinLookupStage(std::move(localStage),
+            return buildHashJoinLookupStage(state,
+                                            std::move(localStage),
                                             localRecordSlot,
                                             localFieldName,
                                             std::move(foreignStage),
@@ -969,8 +976,7 @@ std::pair<SlotId /*matched docs*/, std::unique_ptr<sbe::PlanStage>> buildLookupS
                                             foreignFieldName,
                                             collatorSlot,
                                             nodeId,
-                                            slotIdGenerator,
-                                            allowDiskUse);
+                                            slotIdGenerator);
         default:
             MONGO_UNREACHABLE_TASSERT(5842606);
     }
@@ -1125,7 +1131,8 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
                                                      eqLookupNode->nodeId(),
                                                      ScanCallbacks{});
 
-                return buildLookupStage(eqLookupNode->lookupStrategy,
+                return buildLookupStage(_state,
+                                        eqLookupNode->lookupStrategy,
                                         std::move(localStage),
                                         localDocumentSlot,
                                         eqLookupNode->joinFieldLocal,
@@ -1134,8 +1141,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
                                         eqLookupNode->joinFieldForeign,
                                         collatorSlot,
                                         eqLookupNode->nodeId(),
-                                        _slotIdGenerator,
-                                        _state.allowDiskUse);
+                                        _slotIdGenerator);
             }
             default:
                 MONGO_UNREACHABLE_TASSERT(5842605);
diff --git a/src/mongo/db/stats/counters.cpp b/src/mongo/db/stats/counters.cpp
index eb436737a4a..e354f51f634 100644
--- a/src/mongo/db/stats/counters.cpp
+++ b/src/mongo/db/stats/counters.cpp
@@ -338,6 +338,7 @@ AuthCounter authCounter;
 AggStageCounters aggStageCounters;
 DotsAndDollarsFieldsCounters dotsAndDollarsFieldsCounters;
 QueryEngineCounters queryEngineCounters;
+LookupPushdownCounters lookupPushdownCounters;
 
 OperatorCounters operatorCountersAggExpressions{"operatorCounters.expressions."};
 OperatorCounters operatorCountersMatchExpressions{"operatorCounters.match."};
diff --git a/src/mongo/db/stats/counters.h b/src/mongo/db/stats/counters.h
index 815a6856772..3c01c3eb626 100644
--- a/src/mongo/db/stats/counters.h
+++ b/src/mongo/db/stats/counters.h
@@ -376,6 +376,32 @@ public:
 };
 extern QueryEngineCounters queryEngineCounters;
 
+class LookupPushdownCounters {
+public:
+    LookupPushdownCounters() = default;
+
+    void incrementLookupCounters(OpDebug& debug) {
+        if (debug.pipelineUsesLookup) {
+            totalLookup.increment();
+        }
+        nestedLoopJoin.increment(debug.nestedLoopJoin);
+        indexedLoopJoin.increment(debug.indexedLoopJoin);
+        hashLookup.increment(debug.hashLookup);
+        hashLookupSpillToDisk.increment(debug.hashLookupSpillToDisk);
+    }
+
+    // Counter tracking pipelines that have a lookup stage regardless of the engine used.
+    CounterMetric totalLookup{"query.lookup.pipelineTotalCount"};
+    // Counters for lookup join strategies.
+    CounterMetric nestedLoopJoin{"query.lookup.slotBasedExecutionCounters.nestedLoopJoin"};
+    CounterMetric indexedLoopJoin{"query.lookup.slotBasedExecutionCounters.indexedLoopJoin"};
+    CounterMetric hashLookup{"query.lookup.slotBasedExecutionCounters.hashLookup"};
+    // Counter tracking hashLookup spills in lookup stages that get pushed down.
+    CounterMetric hashLookupSpillToDisk{
+        "query.lookup.slotBasedExecutionCounters.hashLookupSpillToDisk"};
+};
+extern LookupPushdownCounters lookupPushdownCounters;
+
 /**
  * Generic class for counters of expressions inside various MQL statements.
  */