diff options
author | Ivan Fefer <ivan.fefer@mongodb.com> | 2022-09-16 12:21:47 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-09-16 13:14:37 +0000 |
commit | 96cbfd35f635d64d81b58d807bdfe50d90152415 (patch) | |
tree | 097fb53a32d4dc1a94e6a5c9c8259b9d4bf13323 | |
parent | 6680df6423c05f59d20f5eee3e0e8cfcecaaefeb (diff) | |
download | mongo-96cbfd35f635d64d81b58d807bdfe50d90152415.tar.gz |
SERVER-67703 Add query sort metrics to serverStatus
-rw-r--r-- | jstests/noPassthrough/sort_metrics.js | 37 | ||||
-rw-r--r-- | src/mongo/db/curop.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/curop.h | 5 | ||||
-rw-r--r-- | src/mongo/db/curop_metrics.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/pipeline/plan_explainer_pipeline.cpp | 41 | ||||
-rw-r--r-- | src/mongo/db/query/plan_explainer_impl.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/query/plan_summary_stats.h | 9 | ||||
-rw-r--r-- | src/mongo/db/query/plan_summary_stats_visitor.h | 9 | ||||
-rw-r--r-- | src/mongo/db/stats/counters.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/stats/counters.h | 20 |
10 files changed, 95 insertions, 37 deletions
diff --git a/jstests/noPassthrough/sort_metrics.js b/jstests/noPassthrough/sort_metrics.js new file mode 100644 index 00000000000..0883225b506 --- /dev/null +++ b/jstests/noPassthrough/sort_metrics.js @@ -0,0 +1,37 @@ +// Test metrics.query.sort.* ServerStatus counters +(function() { +'use strict'; + +const conn = MongoRunner.runMongod(); +assert.neq(null, conn, "mongod was unable to start up"); + +const db = conn.getDB(jsTestName()); +assert.commandWorked(db.dropDatabase()); +const coll = db.spill_to_disk; + +const memoryLimitMB = 100; +const bigStr = Array(1024 * 1024 + 1).toString(); // 1MB of ',' +for (let i = 0; i < memoryLimitMB + 1; i++) + assert.commandWorked(coll.insert({_id: i, bigStr: i + bigStr, random: Math.random()})); +assert.gt(coll.stats().size, memoryLimitMB * 1024 * 1024); + +const metricsBefore = db.serverStatus().metrics.query.sort; + +const pipeline = [{$sort: {random: 1}}]; +assert.eq(coll.aggregate(pipeline).itcount(), coll.count()); + +const metricsAfter = db.serverStatus().metrics.query.sort; +assert.gt(metricsAfter.spillToDisk, + metricsBefore.spillToDisk, + "Expect metric query.sort.spillToDisk to increment after pipeline " + tojson(pipeline)); +assert.gt( + metricsAfter.totalKeysSorted, + metricsBefore.totalKeysSorted, + "Expect metric query.sort.totalKeysSorted to increment after pipeline " + tojson(pipeline)); +assert.gt( + metricsAfter.totalKeysSorted, + metricsBefore.totalKeysSorted, + "Expect metric query.sort.totalKeysSorted to increment after pipeline " + tojson(pipeline)); + +MongoRunner.stopMongod(conn); +})(); diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index 36f6ba7ebd6..db67644cd1b 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -1439,6 +1439,9 @@ void OpDebug::setPlanSummaryMetrics(const PlanSummaryStats& planSummaryStats) { additiveMetrics.docsExamined = planSummaryStats.totalDocsExamined; hasSortStage = planSummaryStats.hasSortStage; usedDisk = planSummaryStats.usedDisk; + sortSpills = planSummaryStats.sortSpills; + sortTotalDataSizeBytes = planSummaryStats.sortTotalDataSizeBytes; + keysSorted = planSummaryStats.keysSorted; fromMultiPlanner = planSummaryStats.fromMultiPlanner; replanReason = planSummaryStats.replanReason; } diff --git a/src/mongo/db/curop.h b/src/mongo/db/curop.h index 998f2400c73..6418883a605 100644 --- a/src/mongo/db/curop.h +++ b/src/mongo/db/curop.h @@ -242,7 +242,10 @@ public: bool hasSortStage{false}; // true if the query plan involves an in-memory sort - bool usedDisk{false}; // true if the given query used disk + bool usedDisk{false}; // true if the given query used disk + long long sortSpills{0}; // The total number of spills to disk from sort stages + size_t sortTotalDataSizeBytes{0}; // The amount of data we've sorted in bytes + long long keysSorted{0}; // The number of keys that we've sorted. // True if the plan came from the multi-planner (not from the plan cache and not a query with a // single solution). diff --git a/src/mongo/db/curop_metrics.cpp b/src/mongo/db/curop_metrics.cpp index c2d7365852c..512ac4ac0a0 100644 --- a/src/mongo/db/curop_metrics.cpp +++ b/src/mongo/db/curop_metrics.cpp @@ -69,7 +69,7 @@ void recordCurOpMetrics(OperationContext* opCtx) { writeConflictsCounter.increment(n); lookupPushdownCounters.incrementLookupCounters(CurOp::get(opCtx)->debug()); - + sortCounters.incrementSortCounters(debug); queryFrameworkCounters.incrementQueryEngineCounters(CurOp::get(opCtx)); } diff --git a/src/mongo/db/pipeline/plan_explainer_pipeline.cpp b/src/mongo/db/pipeline/plan_explainer_pipeline.cpp index 7128ef650c8..cb26f9822dd 100644 --- a/src/mongo/db/pipeline/plan_explainer_pipeline.cpp +++ b/src/mongo/db/pipeline/plan_explainer_pipeline.cpp @@ -32,26 +32,11 @@ #include "mongo/db/pipeline/plan_explainer_pipeline.h" #include "mongo/db/pipeline/document_source_cursor.h" -#include "mongo/db/pipeline/document_source_facet.h" -#include "mongo/db/pipeline/document_source_lookup.h" -#include "mongo/db/pipeline/document_source_sort.h" -#include "mongo/db/pipeline/document_source_union_with.h" #include "mongo/db/pipeline/plan_executor_pipeline.h" #include "mongo/db/query/explain.h" #include "mongo/db/query/plan_summary_stats_visitor.h" namespace mongo { -/** - * Templatized method to get plan summary stats from document source and aggregate it to 'statsOut'. - */ -template <typename DocSourceType, typename DocSourceStatType> -void collectPlanSummaryStats(const DocSourceType& source, PlanSummaryStats* statsOut) { - auto specificStats = source.getSpecificStats(); - invariant(specificStats); - auto visitor = PlanSummaryStatsVisitor(*statsOut); - specificStats->acceptVisitor(&visitor); -} - const PlanExplainer::ExplainVersion& PlanExplainerPipeline::getVersion() const { static const ExplainVersion kExplainVersion = "1"; @@ -74,27 +59,19 @@ std::string PlanExplainerPipeline::getPlanSummary() const { void PlanExplainerPipeline::getSummaryStats(PlanSummaryStats* statsOut) const { invariant(statsOut); - if (auto docSourceCursor = - dynamic_cast<DocumentSourceCursor*>(_pipeline->getSources().front().get())) { + auto source_it = _pipeline->getSources().begin(); + if (auto docSourceCursor = dynamic_cast<DocumentSourceCursor*>(source_it->get())) { *statsOut = docSourceCursor->getPlanSummaryStats(); - } + ++source_it; + }; - for (auto&& source : _pipeline->getSources()) { + PlanSummaryStatsVisitor visitor(*statsOut); + std::for_each(source_it, _pipeline->getSources().end(), [&](const auto& source) { statsOut->usedDisk = statsOut->usedDisk || source->usedDisk(); - - if (dynamic_cast<DocumentSourceSort*>(source.get())) { - statsOut->hasSortStage = true; - } else if (auto docSourceLookUp = dynamic_cast<DocumentSourceLookUp*>(source.get())) { - collectPlanSummaryStats<DocumentSourceLookUp, DocumentSourceLookupStats>( - *docSourceLookUp, statsOut); - } else if (auto docSourceUnionWith = dynamic_cast<DocumentSourceUnionWith*>(source.get())) { - collectPlanSummaryStats<DocumentSourceUnionWith, UnionWithStats>(*docSourceUnionWith, - statsOut); - } else if (auto docSourceFacet = dynamic_cast<DocumentSourceFacet*>(source.get())) { - collectPlanSummaryStats<DocumentSourceFacet, DocumentSourceFacetStats>(*docSourceFacet, - statsOut); + if (auto specificStats = source->getSpecificStats()) { + specificStats->acceptVisitor(&visitor); } - } + }); if (_nReturned) { statsOut->nReturned = _nReturned; diff --git a/src/mongo/db/query/plan_explainer_impl.cpp b/src/mongo/db/query/plan_explainer_impl.cpp index c5ee9377af2..0b70636945a 100644 --- a/src/mongo/db/query/plan_explainer_impl.cpp +++ b/src/mongo/db/query/plan_explainer_impl.cpp @@ -48,6 +48,7 @@ #include "mongo/db/exec/trial_stage.h" #include "mongo/db/keypattern.h" #include "mongo/db/query/explain.h" +#include "mongo/db/query/plan_summary_stats_visitor.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/record_id_helpers.h" #include "mongo/util/assert_util.h" @@ -680,11 +681,9 @@ void PlanExplainerImpl::getSummaryStats(PlanSummaryStats* statsOut) const { getDocsExamined(stages[i]->stageType(), stages[i]->getSpecificStats()); if (isSortStageType(stages[i]->stageType())) { - statsOut->hasSortStage = true; - auto sortStage = static_cast<const SortStage*>(stages[i]); auto sortStats = static_cast<const SortStats*>(sortStage->getSpecificStats()); - statsOut->usedDisk = sortStats->spills > 0; + PlanSummaryStatsVisitor(*statsOut).visit(sortStats); } if (STAGE_IXSCAN == stages[i]->stageType()) { diff --git a/src/mongo/db/query/plan_summary_stats.h b/src/mongo/db/query/plan_summary_stats.h index a3470609467..59e3985662f 100644 --- a/src/mongo/db/query/plan_summary_stats.h +++ b/src/mongo/db/query/plan_summary_stats.h @@ -91,6 +91,15 @@ struct PlanSummaryStats { // Did this plan use disk space? bool usedDisk = false; + // The total number of spills to disk from sort stages + long long sortSpills = 0; + + // The amount of data we've sorted in bytes + size_t sortTotalDataSizeBytes = 0; + + // The number of keys that we've sorted. + long long keysSorted = 0; + // Did this plan failed during execution? bool planFailed = false; diff --git a/src/mongo/db/query/plan_summary_stats_visitor.h b/src/mongo/db/query/plan_summary_stats_visitor.h index d6bfbdd4f73..70757563ac5 100644 --- a/src/mongo/db/query/plan_summary_stats_visitor.h +++ b/src/mongo/db/query/plan_summary_stats_visitor.h @@ -57,9 +57,15 @@ public: void visit(tree_walker::MaybeConstPtr<true, sbe::IndexScanStats> stats) override final { _summary.totalKeysExamined += stats->keysExamined; } + void visit(tree_walker::MaybeConstPtr<true, sbe::HashAggStats> stats) override final { + _summary.usedDisk |= stats->spilledRecords > 0; + } void visit(tree_walker::MaybeConstPtr<true, SortStats> stats) override final { _summary.hasSortStage = true; _summary.usedDisk = _summary.usedDisk || stats->spills > 0; + _summary.sortSpills += stats->spills; + _summary.sortTotalDataSizeBytes += stats->totalDataSizeBytes; + _summary.keysSorted += stats->keysSorted; } void visit(tree_walker::MaybeConstPtr<true, GroupStats> stats) override final { _summary.usedDisk = _summary.usedDisk || stats->spills > 0; @@ -96,6 +102,9 @@ private: _summary.collectionScansNonTailable += statsIn.collectionScansNonTailable; _summary.hasSortStage |= statsIn.hasSortStage; _summary.usedDisk |= statsIn.usedDisk; + _summary.sortSpills += statsIn.sortSpills; + _summary.sortTotalDataSizeBytes += statsIn.sortTotalDataSizeBytes; + _summary.keysSorted += statsIn.keysSorted; _summary.planFailed |= statsIn.planFailed; _summary.indexesUsed.insert(statsIn.indexesUsed.begin(), statsIn.indexesUsed.end()); } diff --git a/src/mongo/db/stats/counters.cpp b/src/mongo/db/stats/counters.cpp index 3cc52b663aa..67ffd73c359 100644 --- a/src/mongo/db/stats/counters.cpp +++ b/src/mongo/db/stats/counters.cpp @@ -324,6 +324,7 @@ AggStageCounters aggStageCounters; DotsAndDollarsFieldsCounters dotsAndDollarsFieldsCounters; QueryFrameworkCounters queryFrameworkCounters; LookupPushdownCounters lookupPushdownCounters; +SortCounters sortCounters; OperatorCounters operatorCountersAggExpressions{"operatorCounters.expressions."}; OperatorCounters operatorCountersMatchExpressions{"operatorCounters.match."}; diff --git a/src/mongo/db/stats/counters.h b/src/mongo/db/stats/counters.h index 9fe9e7f2126..b1661392574 100644 --- a/src/mongo/db/stats/counters.h +++ b/src/mongo/db/stats/counters.h @@ -388,6 +388,26 @@ public: }; extern LookupPushdownCounters lookupPushdownCounters; +class SortCounters { +public: + SortCounters() = default; + + void incrementSortCounters(const OpDebug& debug) { + sortSpillsCounter.increment(debug.sortSpills); + sortTotalBytesCounter.increment(debug.sortTotalDataSizeBytes); + sortTotalKeysCounter.increment(debug.keysSorted); + } + + // Counters tracking sort stats across all engines + // The total number of spills to disk from sort stages + CounterMetric sortSpillsCounter{"query.sort.spillToDisk"}; + // The number of keys that we've sorted. + CounterMetric sortTotalKeysCounter{"query.sort.totalKeysSorted"}; + // The amount of data we've sorted in bytes + CounterMetric sortTotalBytesCounter{"query.sort.totalBytesSorted"}; +}; +extern SortCounters sortCounters; + /** * Generic class for counters of expressions inside various MQL statements. */ |