diff options
Diffstat (limited to 'src')
22 files changed, 275 insertions, 389 deletions
diff --git a/src/mongo/db/exec/sort.cpp b/src/mongo/db/exec/sort.cpp index eb5486b61ee..a7c4c045234 100644 --- a/src/mongo/db/exec/sort.cpp +++ b/src/mongo/db/exec/sort.cpp @@ -111,7 +111,7 @@ std::unique_ptr<PlanStageStats> SortStage::getStats() { _commonStats.isEOF = isEOF(); std::unique_ptr<PlanStageStats> ret = std::make_unique<PlanStageStats>(_commonStats, STAGE_SORT); - ret->specific = _sortExecutor.stats(); + ret->specific = _sortExecutor.cloneStats(); ret->children.emplace_back(child()->getStats()); return ret; } diff --git a/src/mongo/db/exec/sort.h b/src/mongo/db/exec/sort.h index f437ce0b719..b727771c8bb 100644 --- a/src/mongo/db/exec/sort.h +++ b/src/mongo/db/exec/sort.h @@ -72,12 +72,8 @@ public: std::unique_ptr<PlanStageStats> getStats(); - /** - * Returns nullptr. Stats related to sort execution must be extracted with 'getStats()', since - * they are retrieved on demand from the underlying sort execution machinery. - */ const SpecificStats* getSpecificStats() const final { - return nullptr; + return &_sortExecutor.stats(); } private: @@ -86,8 +82,6 @@ private: SortExecutor _sortExecutor; - SortStats _specificStats; - // Whether or not we have finished loading data into '_sortExecutor'. bool _populated = false; }; diff --git a/src/mongo/db/exec/sort_executor.cpp b/src/mongo/db/exec/sort_executor.cpp index 51e0255a01d..bf2d02a465d 100644 --- a/src/mongo/db/exec/sort_executor.cpp +++ b/src/mongo/db/exec/sort_executor.cpp @@ -55,10 +55,13 @@ SortExecutor::SortExecutor(SortPattern sortPattern, std::string tempDir, bool allowDiskUse) : _sortPattern(std::move(sortPattern)), - _limit(limit), - _maxMemoryUsageBytes(maxMemoryUsageBytes), _tempDir(std::move(tempDir)), - _diskUseAllowed(allowDiskUse) {} + _diskUseAllowed(allowDiskUse) { + _stats.sortPattern = + _sortPattern.serialize(SortPattern::SortKeySerialization::kForExplain).toBson(); + _stats.limit = limit; + _stats.maxMemoryUsageBytes = maxMemoryUsageBytes; +} boost::optional<Document> SortExecutor::getNextDoc() { auto wsm = getNextWsm(); @@ -114,7 +117,7 @@ void SortExecutor::add(Value sortKey, WorkingSetMember data) { } _sorter->add(std::move(sortKey), std::move(data)); - _totalDataSizeBytes += data.getMemUsage(); + _stats.totalDataSizeBytes += data.getMemUsage(); } void SortExecutor::loadingDone() { @@ -123,17 +126,17 @@ void SortExecutor::loadingDone() { _sorter.reset(DocumentSorter::make(makeSortOptions(), Comparator(_sortPattern))); } _output.reset(_sorter->done()); - _wasDiskUsed = _wasDiskUsed || _sorter->usedDisk(); + _stats.wasDiskUsed = _stats.wasDiskUsed || _sorter->usedDisk(); _sorter.reset(); } SortOptions SortExecutor::makeSortOptions() const { SortOptions opts; - if (_limit) { - opts.limit = _limit; + if (_stats.limit) { + opts.limit = _stats.limit; } - opts.maxMemoryUsageBytes = _maxMemoryUsageBytes; + opts.maxMemoryUsageBytes = _stats.maxMemoryUsageBytes; if (_diskUseAllowed) { opts.extSortAllowed = true; opts.tempDir = _tempDir; @@ -142,15 +145,8 @@ SortOptions SortExecutor::makeSortOptions() const { return opts; } -std::unique_ptr<SortStats> SortExecutor::stats() const { - auto stats = std::make_unique<SortStats>(); - stats->sortPattern = - _sortPattern.serialize(SortPattern::SortKeySerialization::kForExplain).toBson(); - stats->limit = _limit; - stats->maxMemoryUsageBytes = _maxMemoryUsageBytes; - stats->totalDataSizeBytes = _totalDataSizeBytes; - stats->wasDiskUsed = _wasDiskUsed; - return stats; +std::unique_ptr<SortStats> SortExecutor::cloneStats() const { + return std::unique_ptr<SortStats>{static_cast<SortStats*>(_stats.clone())}; } } // namespace mongo diff --git a/src/mongo/db/exec/sort_executor.h b/src/mongo/db/exec/sort_executor.h index c0945fe1fd4..085311e19da 100644 --- a/src/mongo/db/exec/sort_executor.h +++ b/src/mongo/db/exec/sort_executor.h @@ -68,20 +68,20 @@ public: * the smallest limit. */ void setLimit(uint64_t limit) { - if (!_limit || limit < _limit) - _limit = limit; + if (!_stats.limit || limit < _stats.limit) + _stats.limit = limit; } uint64_t getLimit() const { - return _limit; + return _stats.limit; } bool hasLimit() const { - return _limit > 0; + return _stats.limit > 0; } bool wasDiskUsed() const { - return _wasDiskUsed; + return _stats.wasDiskUsed; } /** @@ -107,7 +107,11 @@ public: return _isEOF; } - std::unique_ptr<SortStats> stats() const; + const SortStats& stats() const { + return _stats; + } + + std::unique_ptr<SortStats> cloneStats() const; private: using DocumentSorter = Sorter<Value, WorkingSetMember>; @@ -124,18 +128,15 @@ private: SortOptions makeSortOptions() const; - SortPattern _sortPattern; - // A limit of zero is defined as no limit. - uint64_t _limit; - uint64_t _maxMemoryUsageBytes; - std::string _tempDir; - bool _diskUseAllowed = false; + const SortPattern _sortPattern; + const std::string _tempDir; + const bool _diskUseAllowed; std::unique_ptr<DocumentSorter> _sorter; std::unique_ptr<DocumentSorter::Iterator> _output; + SortStats _stats; + bool _isEOF = false; - bool _wasDiskUsed = false; - uint64_t _totalDataSizeBytes = 0u; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp index 803f78422b9..6ad22cce49d 100644 --- a/src/mongo/db/pipeline/document_source_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_cursor.cpp @@ -183,10 +183,6 @@ Value DocumentSourceCursor::serialize(boost::optional<ExplainOptions::Verbosity> verbosity == pExpCtx->explain); MutableDocument out; - out["query"] = Value(_query); - - if (!_sort.isEmpty()) - out["sort"] = Value(_sort); BSONObjBuilder explainStatsBuilder; diff --git a/src/mongo/db/pipeline/document_source_cursor.h b/src/mongo/db/pipeline/document_source_cursor.h index eeb03d1ea55..3db747e66db 100644 --- a/src/mongo/db/pipeline/document_source_cursor.h +++ b/src/mongo/db/pipeline/document_source_cursor.h @@ -83,36 +83,6 @@ public: const boost::intrusive_ptr<ExpressionContext>& pExpCtx, bool trackOplogTimestamp = false); - /* - Record the query that was specified for the cursor this wraps, if - any. - - This should be captured after any optimizations are applied to - the pipeline so that it reflects what is really used. - - This gets used for explain output. - - @param pBsonObj the query to record - */ - void setQuery(const BSONObj& query) { - _query = query; - } - - /* - Record the sort that was specified for the cursor this wraps, if - any. - - This should be captured after any optimizations are applied to - the pipeline so that it reflects what is really used. - - This gets used for explain output. - - @param pBsonObj the sort to record - */ - void setSort(const BSONObj& sort) { - _sort = sort; - } - /** * If subsequent sources need no information from the cursor, the cursor can simply output empty * documents, avoiding the overhead of converting BSONObjs to Documents. @@ -133,6 +103,10 @@ public: return _planSummaryStats; } + bool usedDisk() final { + return _planSummaryStats.usedDisk; + } + protected: DocumentSourceCursor(Collection* collection, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, @@ -183,9 +157,6 @@ private: // Batches results returned from the underlying PlanExecutor. std::deque<Document> _currentBatch; - // BSONObj members must outlive _projection and cursor. - BSONObj _query; - BSONObj _sort; bool _shouldProduceEmptyDocs = false; // The underlying query plan which feeds this pipeline. Must be destroyed while holding the diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp index f8d88637824..2722cc7df7f 100644 --- a/src/mongo/db/pipeline/document_source_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_lookup.cpp @@ -265,10 +265,7 @@ DocumentSource::GetNextResult DocumentSourceLookUp::doGetNext() { objsize <= maxBytes); results.emplace_back(std::move(*result)); } - for (auto&& source : pipeline->getSources()) { - if (source->usedDisk()) - _usedDisk = true; - } + _usedDisk = _usedDisk || pipeline->usedDisk(); MutableDocument output(std::move(inputDoc)); output.setNestedField(_as, Value(std::move(results))); diff --git a/src/mongo/db/pipeline/document_source_single_document_transformation.h b/src/mongo/db/pipeline/document_source_single_document_transformation.h index 6d5daa7171a..8ef435bd776 100644 --- a/src/mongo/db/pipeline/document_source_single_document_transformation.h +++ b/src/mongo/db/pipeline/document_source_single_document_transformation.h @@ -85,10 +85,6 @@ public: return *_parsedTransform; } - bool isSubsetOfProjection(const BSONObj& proj) const { - return _parsedTransform->isSubsetOfProjection(proj); - } - protected: GetNextResult doGetNext() final; void doDispose() final; diff --git a/src/mongo/db/pipeline/parsed_aggregation_projection_node.h b/src/mongo/db/pipeline/parsed_aggregation_projection_node.h index 92ffff5529d..3e77720858a 100644 --- a/src/mongo/db/pipeline/parsed_aggregation_projection_node.h +++ b/src/mongo/db/pipeline/parsed_aggregation_projection_node.h @@ -126,6 +126,11 @@ public: void serialize(boost::optional<ExplainOptions::Verbosity> explain, MutableDocument* output) const; + /** + * Returns true if this node or any child of this node contains a computed field. + */ + bool subtreeContainsComputedFields() const; + protected: // Returns a unique_ptr to a new instance of the implementing class for the given 'fieldName'. virtual std::unique_ptr<ProjectionNode> makeChild(std::string fieldName) const = 0; @@ -182,9 +187,6 @@ private: // Returns nullptr if no such child exists. ProjectionNode* getChild(const std::string& field) const; - // Returns true if this node or any child of this node contains a computed field. - bool subtreeContainsComputedFields() const; - // Our projection semantics are such that all field additions need to be processed in the order // specified. '_orderToProcessAdditionsAndChildren' tracks that order. // diff --git a/src/mongo/db/pipeline/parsed_inclusion_projection.cpp b/src/mongo/db/pipeline/parsed_inclusion_projection.cpp index 23cc83fabe7..b0f990f753e 100644 --- a/src/mongo/db/pipeline/parsed_inclusion_projection.cpp +++ b/src/mongo/db/pipeline/parsed_inclusion_projection.cpp @@ -216,19 +216,29 @@ void ParsedInclusionProjection::parseSubObject(const BSONObj& subObj, } } -bool ParsedInclusionProjection::isSubsetOfProjection(const BSONObj& proj) const { +bool ParsedInclusionProjection::isEquivalentToDependencySet(const BSONObj& deps) const { std::set<std::string> preservedPaths; _root->reportProjectedPaths(&preservedPaths); - for (auto&& includedField : preservedPaths) { - if (!proj.hasField(includedField)) + size_t numDependencies = 0; + for (auto&& dependency : deps) { + if (!dependency.trueValue()) { + // This is not an included field, so move on. + continue; + } + + if (preservedPaths.find(dependency.fieldNameStringData().toString()) == + preservedPaths.end()) { return false; + } + ++numDependencies; + } + + if (numDependencies != preservedPaths.size()) { + return false; } // If the inclusion has any computed fields or renamed fields, then it's not a subset. - std::set<std::string> computedPaths; - StringMap<std::string> renamedPaths; - _root->reportComputedPaths(&computedPaths, &renamedPaths); - return computedPaths.empty() && renamedPaths.empty(); + return !_root->subtreeContainsComputedFields(); } } // namespace parsed_aggregation_projection diff --git a/src/mongo/db/pipeline/parsed_inclusion_projection.h b/src/mongo/db/pipeline/parsed_inclusion_projection.h index a3f205b90b1..67977e41b4b 100644 --- a/src/mongo/db/pipeline/parsed_inclusion_projection.h +++ b/src/mongo/db/pipeline/parsed_inclusion_projection.h @@ -175,11 +175,11 @@ public: Document applyProjection(const Document& inputDoc) const final; /* - * Checks whether the inclusion projection represented by the InclusionNode - * tree is a subset of the object passed in. Projections that have any - * computed or renamed fields are not considered a subset. + * Given 'deps', a BSONObj describing a the dependency set for a pipeline, returns true if this + * is an inclusion projection with no computed paths which includes the exact same set of fields + * as 'deps'. */ - bool isSubsetOfProjection(const BSONObj& proj) const final; + bool isEquivalentToDependencySet(const BSONObj& deps) const; private: /** diff --git a/src/mongo/db/pipeline/parsed_inclusion_projection_test.cpp b/src/mongo/db/pipeline/parsed_inclusion_projection_test.cpp index 6412cbdb15e..89e5c180863 100644 --- a/src/mongo/db/pipeline/parsed_inclusion_projection_test.cpp +++ b/src/mongo/db/pipeline/parsed_inclusion_projection_test.cpp @@ -832,92 +832,114 @@ TEST(InclusionProjectionExecutionTest, ComputedFieldShouldReplaceNestedArrayForN } // -// Detection of subset projection. +// Detection of equivalency to the dependency set. // -TEST(InclusionProjectionExecutionTest, ShouldDetectSubsetForIdenticalProjection) { +TEST(InclusionProjectionExecutionTest, ShouldDetectEquivalenceForIdenticalProjection) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a" << true << "b" << true)); - auto proj = BSON("_id" << false << "a" << true << "b" << true); + auto proj = BSON("_id" << true << "a" << true << "b" << true); - ASSERT_TRUE(inclusion.isSubsetOfProjection(proj)); + ASSERT_TRUE(inclusion.isEquivalentToDependencySet(proj)); } -TEST(InclusionProjectionExecutionTest, ShouldDetectSubsetForSupersetProjection) { +TEST(InclusionProjectionExecutionTest, ShouldNotDetectEquivalenceForSupersetProjection) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a" << true << "b" << true)); - auto proj = BSON("_id" << false << "a" << true << "b" << true << "c" << true); + auto proj = BSON("_id" << true << "a" << true << "b" << true << "c" << true); - ASSERT_TRUE(inclusion.isSubsetOfProjection(proj)); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); } -TEST(InclusionProjectionExecutionTest, ShouldDetectSubsetForIdenticalNestedProjection) { +TEST(InclusionProjectionExecutionTest, ShouldDetectEquivalenceForIdenticalNestedProjection) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a.b" << true)); - auto proj = BSON("_id" << false << "a.b" << true); + auto proj = BSON("_id" << true << "a.b" << true); - ASSERT_TRUE(inclusion.isSubsetOfProjection(proj)); + ASSERT_TRUE(inclusion.isEquivalentToDependencySet(proj)); } -TEST(InclusionProjectionExecutionTest, ShouldDetectSubsetForSupersetProjectionWithNestedFields) { +TEST(InclusionProjectionExecutionTest, + ShouldNotDetectEquivalenceForSupersetProjectionWithNestedFields) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a" << true << "c" << BSON("d" << true))); - auto proj = BSON("_id" << false << "a" << true << "b" << true << "c.d" << true); + auto proj = BSON("_id" << true << "a" << true << "b" << true << "c.d" << true); - ASSERT_TRUE(inclusion.isSubsetOfProjection(proj)); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); } -TEST(InclusionProjectionExecutionTest, ShouldDetectNonSubsetForProjectionWithMissingFields) { +TEST(InclusionProjectionExecutionTest, ShouldNotDetectEquivalenceForProjectionWithMissingFields) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a" << true << "b" << true)); - auto proj = BSON("_id" << false << "a" << true); - ASSERT_FALSE(inclusion.isSubsetOfProjection(proj)); + auto proj = BSON("_id" << true << "a" << true); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); - proj = BSON("_id" << false << "a" << true << "c" << true); - ASSERT_FALSE(inclusion.isSubsetOfProjection(proj)); + proj = BSON("_id" << true << "a" << true << "c" << true); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); } TEST(InclusionProjectionExecutionTest, - ShouldDetectNonSubsetForSupersetProjectionWithoutComputedFields) { + ShouldNotDetectEquivalenceForSupersetProjectionWithoutComputedFields) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a" << true << "b" << true << "c" << BSON("$literal" << 1))); auto proj = BSON("_id" << false << "a" << true << "b" << true); - ASSERT_FALSE(inclusion.isSubsetOfProjection(proj)); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); } -TEST(InclusionProjectionExecutionTest, ShouldDetectNonSubsetForProjectionWithMissingNestedFields) { +TEST(InclusionProjectionExecutionTest, + ShouldNotDetectEquivalenceForProjectionWithMissingNestedFields) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a.b" << true << "a.c" << true)); auto proj = BSON("_id" << false << "a.b" << true); - ASSERT_FALSE(inclusion.isSubsetOfProjection(proj)); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); } -TEST(InclusionProjectionExecutionTest, ShouldDetectNonSubsetForProjectionWithRenamedFields) { +TEST(InclusionProjectionExecutionTest, ShouldNotDetectEquivalenceForProjectionWithRenamedFields) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a" << "$b")); auto proj = BSON("_id" << false << "b" << true); - ASSERT_FALSE(inclusion.isSubsetOfProjection(proj)); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); } -TEST(InclusionProjectionExecutionTest, ShouldDetectNonSubsetForProjectionWithMissingIdField) { +TEST(InclusionProjectionExecutionTest, ShouldNotDetectEquivalenceForProjectionWithMissingIdField) { auto inclusion = makeInclusionProjectionWithDefaultPolicies(); inclusion.parse(BSON("a" << true)); auto proj = BSON("a" << true); - ASSERT_FALSE(inclusion.isSubsetOfProjection(proj)); + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); +} + +TEST(InclusionProjectionExecutionTest, + ShouldNotDetectEquivalenceIfDependenciesExplicitlyExcludeId) { + auto inclusion = makeInclusionProjectionWithDefaultPolicies(); + inclusion.parse(BSON("a" << true)); + + auto proj = BSON("_id" << false << "a" << true); + + ASSERT_FALSE(inclusion.isEquivalentToDependencySet(proj)); +} + +TEST(InclusionProjectionExecutionTest, + ShouldDetectEquivalenceIfBothDepsAndProjExplicitlyExcludeId) { + auto inclusion = makeInclusionProjectionWithDefaultPolicies(); + inclusion.parse(BSON("_id" << false << "a" << true)); + + auto proj = BSON("_id" << false << "a" << true); + + ASSERT_TRUE(inclusion.isEquivalentToDependencySet(proj)); } } // namespace diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp index a0e6996829e..0ce917467f5 100644 --- a/src/mongo/db/pipeline/pipeline_d.cpp +++ b/src/mongo/db/pipeline/pipeline_d.cpp @@ -66,6 +66,7 @@ #include "mongo/db/pipeline/document_source_sample_from_random_cursor.h" #include "mongo/db/pipeline/document_source_single_document_transformation.h" #include "mongo/db/pipeline/document_source_sort.h" +#include "mongo/db/pipeline/parsed_inclusion_projection.h" #include "mongo/db/pipeline/pipeline.h" #include "mongo/db/query/collation/collator_interface.h" #include "mongo/db/query/get_executor.h" @@ -187,10 +188,9 @@ StatusWith<unique_ptr<PlanExecutor, PlanExecutor::Deleter>> createRandomCursorEx } StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> attemptToGetExecutor( - OperationContext* opCtx, + const intrusive_ptr<ExpressionContext>& expCtx, Collection* collection, const NamespaceString& nss, - const intrusive_ptr<ExpressionContext>& pExpCtx, BSONObj queryObj, BSONObj projectionObj, const QueryMetadataBitSet& metadataRequested, @@ -201,7 +201,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> attemptToGetExe const size_t plannerOpts, const MatchExpressionParser::AllowedFeatureSet& matcherFeatures) { auto qr = std::make_unique<QueryRequest>(nss); - qr->setTailableMode(pExpCtx->tailableMode); + qr->setTailableMode(expCtx->tailableMode); qr->setFilter(queryObj); qr->setProj(projectionObj); qr->setSort(sortObj); @@ -214,12 +214,12 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> attemptToGetExe // The collation on the ExpressionContext has been resolved to either the user-specified // collation or the collection default. This BSON should never be empty even if the resolved // collator is simple. - qr->setCollation(pExpCtx->getCollatorBSON()); + qr->setCollation(expCtx->getCollatorBSON()); - const ExtensionsCallbackReal extensionsCallback(pExpCtx->opCtx, &nss); + const ExtensionsCallbackReal extensionsCallback(expCtx->opCtx, &nss); auto cq = CanonicalQuery::canonicalize( - opCtx, std::move(qr), pExpCtx, extensionsCallback, matcherFeatures); + expCtx->opCtx, std::move(qr), expCtx, extensionsCallback, matcherFeatures); if (!cq.isOK()) { // Return an error instead of uasserting, since there are cases where the combination of @@ -248,7 +248,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> attemptToGetExe // example, if we have a document {a: [1,2]} and group by "a" a DISTINCT_SCAN on an "a" // index would produce one result for '1' and another for '2', which would be incorrect. auto distinctExecutor = - getExecutorDistinct(opCtx, + getExecutorDistinct(expCtx->opCtx, collection, plannerOpts | QueryPlannerParams::STRICT_DISTINCT_ONLY, &parsedDistinct); @@ -264,7 +264,8 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> attemptToGetExe } bool permitYield = true; - return getExecutorFind(opCtx, collection, std::move(cq.getValue()), permitYield, plannerOpts); + return getExecutorFind( + expCtx->opCtx, collection, std::move(cq.getValue()), permitYield, plannerOpts); } /** @@ -361,13 +362,15 @@ PipelineD::buildInnerQueryExecutor(Collection* collection, // TODO SERVER-37453 this should no longer be necessary when we no don't need locks // to destroy a PlanExecutor. auto deps = pipeline->getDependencies(DepsTracker::kNoMetadata); + const bool shouldProduceEmptyDocs = deps.hasNoRequirements(); auto attachExecutorCallback = - [deps](Collection* collection, - std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, - Pipeline* pipeline) { + [shouldProduceEmptyDocs]( + Collection* collection, + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, + Pipeline* pipeline) { auto cursor = DocumentSourceCursor::create( collection, std::move(exec), pipeline->getContext()); - addCursorSource(pipeline, std::move(cursor), std::move(deps)); + addCursorSource(pipeline, std::move(cursor), shouldProduceEmptyDocs); }; return std::make_pair(std::move(attachExecutorCallback), std::move(exec)); } @@ -485,24 +488,10 @@ PipelineD::buildInnerQueryExecutorGeneric(Collection* collection, } } - // Find the set of fields in the source documents depended on by this pipeline. - DepsTracker deps = pipeline->getDependencies(DocumentSourceMatch::isTextQuery(queryObj) - ? DepsTracker::kOnlyTextScore - : DepsTracker::kNoMetadata); - - BSONObj projForQuery = deps.toProjectionWithoutMetadata(); - boost::intrusive_ptr<DocumentSourceSort> sortStage; boost::intrusive_ptr<DocumentSourceGroup> groupStage; std::tie(sortStage, groupStage) = getSortAndGroupStagesFromPipeline(pipeline->_sources); - BSONObj sortObj; - if (sortStage) { - sortObj = sortStage->getSortKeyPattern() - .serialize(SortPattern::SortKeySerialization::kForPipelineSerialization) - .toBson(); - } - std::unique_ptr<GroupFromFirstDocumentTransformation> rewrittenGroupStage; if (groupStage) { rewrittenGroupStage = groupStage->rewriteGroupAsTransformOnFirstDocument(); @@ -525,21 +514,26 @@ PipelineD::buildInnerQueryExecutorGeneric(Collection* collection, // layer, but that is handled elsewhere. const auto limit = extractLimitForPushdown(pipeline); + auto metadataAvailable = DocumentSourceMatch::isTextQuery(queryObj) + ? DepsTracker::kOnlyTextScore + : DepsTracker::kNoMetadata; + // Create the PlanExecutor. - auto exec = uassertStatusOK(prepareExecutor(expCtx->opCtx, + BSONObj projForQuery; + bool shouldProduceEmptyDocs = false; + auto exec = uassertStatusOK(prepareExecutor(expCtx, collection, nss, pipeline, - expCtx, sortStage, std::move(rewrittenGroupStage), - deps, + metadataAvailable, queryObj, limit, aggRequest, Pipeline::kAllowedMatcherFeatures, - &sortObj, - &projForQuery)); + &projForQuery, + &shouldProduceEmptyDocs)); if (!projForQuery.isEmpty() && !sources.empty()) { @@ -547,8 +541,14 @@ PipelineD::buildInnerQueryExecutorGeneric(Collection* collection, // projection generated by the dependency optimization. auto proj = dynamic_cast<DocumentSourceSingleDocumentTransformation*>(sources.front().get()); - if (proj && proj->isSubsetOfProjection(projForQuery)) { - sources.pop_front(); + if (proj && + proj->getType() == TransformerInterface::TransformerType::kInclusionProjection) { + auto&& inclusionProj = + static_cast<const parsed_aggregation_projection::ParsedInclusionProjection&>( + proj->getTransformer()); + if (inclusionProj.isEquivalentToDependencySet(projForQuery)) { + sources.pop_front(); + } } } @@ -556,13 +556,13 @@ PipelineD::buildInnerQueryExecutorGeneric(Collection* collection, const bool trackOplogTS = (pipeline->peekFront() && pipeline->peekFront()->constraints().isChangeStreamStage()); - auto attachExecutorCallback = [deps, queryObj, sortObj, projForQuery, trackOplogTS]( + auto attachExecutorCallback = [shouldProduceEmptyDocs, trackOplogTS]( Collection* collection, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, Pipeline* pipeline) { auto cursor = DocumentSourceCursor::create( collection, std::move(exec), pipeline->getContext(), trackOplogTS); - addCursorSource(pipeline, std::move(cursor), std::move(deps), queryObj, sortObj); + addCursorSource(pipeline, std::move(cursor), shouldProduceEmptyDocs); }; return std::make_pair(std::move(attachExecutorCallback), std::move(exec)); } @@ -582,8 +582,6 @@ PipelineD::buildInnerQueryExecutorGeoNear(Collection* collection, const auto geoNearStage = dynamic_cast<DocumentSourceGeoNear*>(sources.front().get()); invariant(geoNearStage); - auto deps = pipeline->getDependencies(DepsTracker::kAllGeoNearData); - // If the user specified a "key" field, use that field to satisfy the "near" query. Otherwise, // look for a geo-indexed field in 'collection' that can. auto nearFieldName = @@ -594,28 +592,24 @@ PipelineD::buildInnerQueryExecutorGeoNear(Collection* collection, // Create a PlanExecutor whose query is the "near" predicate on 'nearFieldName' combined with // the optional "query" argument in the $geoNear stage. BSONObj fullQuery = geoNearStage->asNearQuery(nearFieldName); - BSONObj proj = deps.toProjectionWithoutMetadata(); - BSONObj sortFromQuerySystem; - auto exec = uassertStatusOK(prepareExecutor(expCtx->opCtx, + + BSONObj proj; + bool shouldProduceEmptyDocs = false; + auto exec = uassertStatusOK(prepareExecutor(expCtx, collection, nss, pipeline, - expCtx, nullptr, /* sortStage */ nullptr, /* rewrittenGroupStage */ - deps, + DepsTracker::kAllGeoNearData, std::move(fullQuery), boost::none, /* limit */ aggRequest, Pipeline::kGeoNearMatcherFeatures, - &sortFromQuerySystem, - &proj)); + &proj, + &shouldProduceEmptyDocs)); - invariant(sortFromQuerySystem.isEmpty(), - str::stream() << "Unexpectedly got the following sort from the query system: " - << sortFromQuerySystem.jsonString()); - - auto attachExecutorCallback = [deps, + auto attachExecutorCallback = [shouldProduceEmptyDocs, distanceField = geoNearStage->getDistanceField(), locationField = geoNearStage->getLocationField(), distanceMultiplier = @@ -629,7 +623,7 @@ PipelineD::buildInnerQueryExecutorGeoNear(Collection* collection, distanceField, locationField, distanceMultiplier); - addCursorSource(pipeline, std::move(cursor), std::move(deps)); + addCursorSource(pipeline, std::move(cursor), shouldProduceEmptyDocs); }; // Remove the initial $geoNear; it will be replaced by $geoNearCursor. sources.pop_front(); @@ -637,45 +631,23 @@ PipelineD::buildInnerQueryExecutorGeoNear(Collection* collection, } StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prepareExecutor( - OperationContext* opCtx, + const intrusive_ptr<ExpressionContext>& expCtx, Collection* collection, const NamespaceString& nss, Pipeline* pipeline, - const intrusive_ptr<ExpressionContext>& expCtx, const boost::intrusive_ptr<DocumentSourceSort>& sortStage, std::unique_ptr<GroupFromFirstDocumentTransformation> rewrittenGroupStage, - const DepsTracker& deps, + QueryMetadataBitSet metadataAvailable, const BSONObj& queryObj, boost::optional<long long> limit, const AggregationRequest* aggRequest, const MatchExpressionParser::AllowedFeatureSet& matcherFeatures, - BSONObj* sortObj, - BSONObj* projectionObj) { - // The query system has the potential to use an index to provide a non-blocking sort and/or to - // use the projection to generate a covered plan. If this is possible, it is more efficient to - // let the query system handle those parts of the pipeline. If not, it is more efficient to use - // a $sort and/or a $project. Thus, we will determine whether the query system can - // provide a non-blocking sort or a covered projection before we commit to a PlanExecutor. - // - // To determine if the query system can provide a non-blocking sort, we pass the - // NO_BLOCKING_SORT planning option, meaning 'getExecutor' will not produce a PlanExecutor if - // the query system would use a blocking sort stage. - // - // To determine if the query system can provide a covered projection, we pass the - // NO_UNCOVERED_PROJECTS planning option, meaning 'getExecutor' will not produce a PlanExecutor - // if the query system would need to fetch the document to do the projection. The following - // logic uses the above strategies, with multiple calls to 'attemptToGetExecutor' to determine - // the most efficient way to handle the $sort and $project stages. - // - // LATER - We should attempt to determine if the results from the query are returned in some - // order so we can then apply other optimizations there are tickets for, such as SERVER-4507. - size_t plannerOpts = QueryPlannerParams::DEFAULT | QueryPlannerParams::NO_BLOCKING_SORT; + BSONObj* projectionObj, + bool* hasNoRequirements) { + invariant(projectionObj); + invariant(hasNoRequirements); - if (deps.hasNoRequirements()) { - // If we don't need any fields from the input document, performing a count is faster, and - // will output empty documents, which is okay. - plannerOpts |= QueryPlannerParams::IS_COUNT; - } + size_t plannerOpts = QueryPlannerParams::DEFAULT; if (pipeline->peekFront() && pipeline->peekFront()->constraints().isChangeStreamStage()) { invariant(expCtx->tailableMode == TailableModeEnum::kTailableAndAwaitData); @@ -687,6 +659,43 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep expCtx->use42ChangeStreamSortKeys = true; } + // If there is a sort stage eligible for pushdown, serialize its SortPattern to a BSONObj. The + // BSONObj format is currently necessary to request that the sort is computed by the query layer + // inside the inner PlanExecutor. We also remove the $sort stage from the Pipeline, since it + // will be handled instead by PlanStage execution. + BSONObj sortObj; + if (sortStage && canSortBePushedDown(sortStage->getSortKeyPattern())) { + sortObj = sortStage->getSortKeyPattern() + .serialize(SortPattern::SortKeySerialization::kForPipelineSerialization) + .toBson(); + + // If the $sort has a coalesced $limit, then we push it down as well. Since the $limit was + // after a $sort in the pipeline, it should not have been provided by the caller. + invariant(!limit); + limit = sortStage->getLimit(); + + pipeline->popFrontWithName(DocumentSourceSort::kStageName); + } + + // Perform dependency analysis. In order to minimize the dependency set, we only analyze the + // stages that remain in the pipeline after pushdown. In particular, any dependencies for a + // $match or $sort pushed down into the query layer will not be reflected here. + auto deps = pipeline->getDependencies(metadataAvailable); + *hasNoRequirements = deps.hasNoRequirements(); + *projectionObj = deps.toProjectionWithoutMetadata(); + + // If we're pushing down a sort, and a merge will be required later, then we need the query + // system to produce sortKey metadata. + if (!sortObj.isEmpty() && expCtx->needsMerge) { + deps.setNeedsMetadata(DocumentMetadataFields::kSortKey, true); + } + + if (deps.hasNoRequirements()) { + // This query might be eligible for count optimizations, since the remaining stages in the + // pipeline don't actually need to read any data produced by the query execution layer. + plannerOpts |= QueryPlannerParams::IS_COUNT; + } + if (rewrittenGroupStage) { BSONObj emptySort; @@ -695,14 +704,13 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep // attemptToGetExecutor() calls below) causes getExecutorDistinct() to ignore some otherwise // valid DISTINCT_SCAN plans, so we pass the projection and exclude the // NO_UNCOVERED_PROJECTIONS planner parameter. - auto swExecutorGrouped = attemptToGetExecutor(opCtx, + auto swExecutorGrouped = attemptToGetExecutor(expCtx, collection, nss, - expCtx, queryObj, *projectionObj, deps.metadataDeps(), - sortObj ? *sortObj : emptySort, + sortObj, boost::none, /* limit */ rewrittenGroupStage->groupId(), aggRequest, @@ -736,10 +744,28 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep } } - const BSONObj emptyProjection; - const BSONObj metaSortProjection = BSON("$sortKey" << BSON("$meta" - << "sortKey")); - + // Unlike stages such as $match and limit which always get pushed down into the inner + // PlanExecutor when present at the front of the pipeline, 'projectionObj' may not always be + // pushed down. (Note that 'projectionObj' is generated based on the dependency set, and + // therefore is not always identical to a $project stage in the pipeline.) The query system has + // the potential to use an index produce a covered plan, computing the projection based on index + // keys rather than documents fetched from the collection. If this is possible, it is more + // efficient to let the query system handle the projection, since covered plans typically have a + // large performance advantage. If not, it is more currently more efficient to compute the + // projection in the agg layer. + // + // To determine if the query system can provide a covered projection, we pass the + // NO_UNCOVERED_PROJECTIONS planning option, meaning 'getExecutor' will not produce a + // PlanExecutor if the query system would need to fetch the document to do the projection. If + // planning fails due to the NO_COVERED_PROJECTIONS option, then we invoke the planner a second + // time without passing 'projectionObj', resulting in a plan where the agg layer handles the + // projection. + // + // The only way to get meta information (e.g. the text score) is to let the query system handle + // the projection. In all other cases, unless the query system can do an index-covered + // projection and avoid going to the raw record at all, it is faster to have the agg system + // perform the projection. + // // TODO SERVER-42905: It should be possible to push down all eligible projections to the query // layer. This code assumes that metadata is passed from the query layer to the DocumentSource // layer via a projection, which is no longer true. @@ -747,100 +773,14 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep plannerOpts |= QueryPlannerParams::NO_UNCOVERED_PROJECTIONS; } - SortPattern userSortPattern(*sortObj, expCtx); - if (sortStage && canSortBePushedDown(userSortPattern)) { - QueryMetadataBitSet needsSortKey; - needsSortKey.set(DocumentMetadataFields::MetaType::kSortKey); - - // If the $sort has a coalesced $limit, then we push it down as well. Since the $limit was - // after a $sort in the pipeline, it should not have been provided by the caller. - invariant(!limit); - auto limitFollowingSort = sortStage->getLimit(); - - // See if the query system can provide a non-blocking sort. - auto swExecutorSort = - attemptToGetExecutor(opCtx, - collection, - nss, - expCtx, - queryObj, - BSONObj(), // empty projection - expCtx->needsMerge ? needsSortKey : DepsTracker::kNoMetadata, - *sortObj, - limitFollowingSort, - boost::none, /* groupIdForDistinctScan */ - aggRequest, - plannerOpts, - matcherFeatures); - - if (swExecutorSort.isOK()) { - // Success! Now see if the query system can also cover the projection. - auto swExecutorSortAndProj = - attemptToGetExecutor(opCtx, - collection, - nss, - expCtx, - queryObj, - *projectionObj, - deps.metadataDeps(), - *sortObj, - limitFollowingSort, - boost::none, /* groupIdForDistinctScan */ - aggRequest, - plannerOpts, - matcherFeatures); - - std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec; - if (swExecutorSortAndProj.isOK()) { - // Success! We have a non-blocking sort and a covered projection. - exec = std::move(swExecutorSortAndProj.getValue()); - } else if (swExecutorSortAndProj != ErrorCodes::NoQueryExecutionPlans) { - - return swExecutorSortAndProj.getStatus().withContext( - "Failed to determine whether query system can provide a " - "covered projection in addition to a non-blocking sort"); - } else { - // The query system couldn't cover the projection. - *projectionObj = BSONObj(); - exec = std::move(swExecutorSort.getValue()); - } - - // We know the sort (and any $limit which coalesced with the $sort) is being handled by - // the query system, so remove the $sort stage. - pipeline->_sources.pop_front(); - - return std::move(exec); - } else if (swExecutorSort != ErrorCodes::NoQueryExecutionPlans) { - return swExecutorSort.getStatus().withContext( - "Failed to determine whether query system can provide a non-blocking sort"); - } - } - - // Either there was no $sort stage, or the query system could not provide a non-blocking - // sort. - *sortObj = BSONObj(); - - // Since the DocumentSource layer will perform the sort, remove any dependencies we have on the - // query layer for a sort key. - QueryMetadataBitSet metadataDepsWithoutSortKey = deps.metadataDeps(); - metadataDepsWithoutSortKey[DocumentMetadataFields::kSortKey] = false; - if (!metadataDepsWithoutSortKey.any()) { - // A sort key requirement would have prevented us from being able to add this parameter - // before, but now we know the query system won't cover the sort, so we will be able to - // compute the sort key ourselves during the $sort stage, and thus don't need a query - // projection to do so. - plannerOpts |= QueryPlannerParams::NO_UNCOVERED_PROJECTIONS; - } - - // See if the query system can cover the projection. - auto swExecutorProj = attemptToGetExecutor(opCtx, + // See if the query layer can use the projection to produce a covered plan. + auto swExecutorProj = attemptToGetExecutor(expCtx, collection, nss, - expCtx, queryObj, *projectionObj, - metadataDepsWithoutSortKey, - *sortObj, + deps.metadataDeps(), + sortObj, limit, boost::none, /* groupIdForDistinctScan */ aggRequest, @@ -854,18 +794,18 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep "Failed to determine whether query system can provide a covered projection"); } - // The query system couldn't provide a covered or simple uncovered projection. Do no projections - // and request no metadata from the query layer. + // The query system couldn't generate a covered plan for the projection. Make another attempt + // without the projection. We need not request any metadata: if there are metadata dependencies, + // then we always push the projection down to the query layer (which is implemented by + // refraining from setting the 'NO_UNCOVERED_PROJECTIONS' parameter). *projectionObj = BSONObj(); - // If this doesn't work, nothing will. - return attemptToGetExecutor(opCtx, + return attemptToGetExecutor(expCtx, collection, nss, - expCtx, queryObj, *projectionObj, DepsTracker::kNoMetadata, - *sortObj, + sortObj, limit, boost::none, /* groupIdForDistinctScan */ aggRequest, @@ -875,16 +815,12 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep void PipelineD::addCursorSource(Pipeline* pipeline, boost::intrusive_ptr<DocumentSourceCursor> cursor, - DepsTracker deps, - const BSONObj& queryObj, - const BSONObj& sortObj) { + bool shouldProduceEmptyDocs) { // Add the cursor to the pipeline first so that it's correctly disposed of as part of the // pipeline if an exception is thrown during this method. pipeline->addInitialSource(cursor); - cursor->setQuery(queryObj); - cursor->setSort(sortObj); - if (deps.hasNoRequirements()) { + if (shouldProduceEmptyDocs) { cursor->shouldProduceEmptyDocs(); } } diff --git a/src/mongo/db/pipeline/pipeline_d.h b/src/mongo/db/pipeline/pipeline_d.h index 836b84301b5..20fe571423c 100644 --- a/src/mongo/db/pipeline/pipeline_d.h +++ b/src/mongo/db/pipeline/pipeline_d.h @@ -170,40 +170,42 @@ private: * an index to provide a more efficient sort or projection, the sort and/or projection will be * incorporated into the PlanExecutor. * - * 'sortObj' will be set to an empty object if the query system cannot provide a non-blocking - * sort, and 'projectionObj' will be set to an empty object if the query system cannot provide a - * covered projection. - * * Set 'rewrittenGroupStage' when the pipeline uses $match+$sort+$group stages that are * compatible with a DISTINCT_SCAN plan that visits the first document in each group * (SERVER-9507). + * + * This function computes the dependencies of 'pipeline' and attempts to push the dependency set + * down to the query layer as a projection. If the dependency set indeed results in a projection + * being pushed down to the query layer, this projection is returned in 'projectionObj'. If no + * such projection can be pushed down, then 'projectionObj' is set to the empty BSONObj. This + * can happen if the query system cannot provide a covered projection. + * + * Sets the 'hasNoRequirements' out-parameter based on whether the dependency set is both finite + * and empty. In this case, the query has count semantics. */ static StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> prepareExecutor( - OperationContext* opCtx, + const boost::intrusive_ptr<ExpressionContext>& expCtx, Collection* collection, const NamespaceString& nss, Pipeline* pipeline, - const boost::intrusive_ptr<ExpressionContext>& expCtx, const boost::intrusive_ptr<DocumentSourceSort>& sortStage, std::unique_ptr<GroupFromFirstDocumentTransformation> rewrittenGroupStage, - const DepsTracker& deps, + QueryMetadataBitSet metadataAvailable, const BSONObj& queryObj, boost::optional<long long> limit, const AggregationRequest* aggRequest, const MatchExpressionParser::AllowedFeatureSet& matcherFeatures, - BSONObj* sortObj, - BSONObj* projectionObj); + BSONObj* projectionObj, + bool* hasNoRequirements); /** - * Adds 'cursor' to the front of 'pipeline', using 'deps' to inform the cursor of its - * dependencies. If specified, 'queryObj', 'sortObj' and 'projectionObj' are passed to the - * cursor for explain reporting. + * Adds 'cursor' to the front of 'pipeline'. If 'shouldProduceEmptyDocs' is true, then we inform + * 'cursor' that this is a count scenario -- the dependency set is fully known and is empty. In + * this case, 'cursor' can return a sequence of empty documents for the caller to count. */ static void addCursorSource(Pipeline* pipeline, boost::intrusive_ptr<DocumentSourceCursor> cursor, - DepsTracker deps, - const BSONObj& queryObj = BSONObj(), - const BSONObj& sortObj = BSONObj()); + bool shouldProduceEmptyDocs); }; } // namespace mongo diff --git a/src/mongo/db/pipeline/transformer_interface.h b/src/mongo/db/pipeline/transformer_interface.h index d82970702b3..d726b0c91ff 100644 --- a/src/mongo/db/pipeline/transformer_interface.h +++ b/src/mongo/db/pipeline/transformer_interface.h @@ -67,19 +67,5 @@ public: */ virtual Document serializeTransformation( boost::optional<ExplainOptions::Verbosity> explain) const = 0; - - /** - * Returns true if this transformer is an inclusion projection and is a subset of - * 'proj', which must be a valid projection specification. For example, if this - * TransformerInterface represents the inclusion projection - * - * {a: 1, b: 1, c: 1} - * - * then it is a subset of the projection {a: 1, c: 1}, and this function returns - * true. - */ - virtual bool isSubsetOfProjection(const BSONObj& proj) const { - return false; - } }; } // namespace mongo diff --git a/src/mongo/db/query/explain.cpp b/src/mongo/db/query/explain.cpp index 665d20d549a..3466e16240c 100644 --- a/src/mongo/db/query/explain.cpp +++ b/src/mongo/db/query/explain.cpp @@ -31,7 +31,6 @@ #include "mongo/db/query/explain.h" -#include "mongo/base/owned_pointer_vector.h" #include "mongo/bson/util/builder.h" #include "mongo/db/exec/cached_plan.h" #include "mongo/db/exec/collection_scan.h" @@ -42,6 +41,7 @@ #include "mongo/db/exec/multi_plan.h" #include "mongo/db/exec/near.h" #include "mongo/db/exec/pipeline_proxy.h" +#include "mongo/db/exec/sort.h" #include "mongo/db/exec/text.h" #include "mongo/db/exec/working_set_common.h" #include "mongo/db/keypattern.h" @@ -983,6 +983,10 @@ void Explain::getSummaryStats(const PlanExecutor& exec, PlanSummaryStats* statsO if (STAGE_SORT == stages[i]->stageType()) { statsOut->hasSortStage = true; + + auto sortStage = static_cast<const SortStage*>(stages[i]); + auto sortStats = static_cast<const SortStats*>(sortStage->getSpecificStats()); + statsOut->usedDisk = sortStats->wasDiskUsed; } if (STAGE_IXSCAN == stages[i]->stageType()) { diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index 9be58ae42b2..f610b940db4 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -1021,6 +1021,14 @@ bool turnIxscanIntoCount(QuerySolution* soln) { return false; } + // Since count scans return no data, they are always forward scans. Index scans, on the other + // hand, may need to scan the index in reverse order in order to obtain a sort. If the index + // scan direction is backwards, then we need to swap the start and end of the count scan bounds. + if (isn->direction < 0) { + startKey.swap(endKey); + std::swap(startKeyInclusive, endKeyInclusive); + } + // Make the count node that we replace the fetch + ixscan with. CountScanNode* csn = new CountScanNode(isn->index); csn->startKey = startKey; diff --git a/src/mongo/db/query/planner_analysis.cpp b/src/mongo/db/query/planner_analysis.cpp index 27405653ae7..5a0d1c8a18c 100644 --- a/src/mongo/db/query/planner_analysis.cpp +++ b/src/mongo/db/query/planner_analysis.cpp @@ -619,12 +619,6 @@ QuerySolutionNode* QueryPlannerAnalysis::analyzeSort(const CanonicalQuery& query // If we're here, we need to add a sort stage. - // If we're not allowed to put a blocking sort in, bail out. - if (params.options & QueryPlannerParams::NO_BLOCKING_SORT) { - delete solnRoot; - return nullptr; - } - if (!solnRoot->fetched()) { const bool sortIsCovered = std::all_of(sortObj.begin(), sortObj.end(), [solnRoot](BSONElement e) { diff --git a/src/mongo/db/query/query_planner.cpp b/src/mongo/db/query/query_planner.cpp index 56bbe55dfe4..46d90750b5c 100644 --- a/src/mongo/db/query/query_planner.cpp +++ b/src/mongo/db/query/query_planner.cpp @@ -108,9 +108,6 @@ string optionString(size_t options) { case QueryPlannerParams::INCLUDE_SHARD_FILTER: ss << "INCLUDE_SHARD_FILTER "; break; - case QueryPlannerParams::NO_BLOCKING_SORT: - ss << "NO_BLOCKING_SORT "; - break; case QueryPlannerParams::INDEX_INTERSECTION: ss << "INDEX_INTERSECTION "; break; @@ -854,7 +851,7 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan( if (0 == out.size() && relevantIndices.front().type != IndexType::INDEX_WILDCARD) { // Push hinted index solution to output list if found. It is possible to end up without // a solution in the case where a filtering QueryPlannerParams argument, such as - // NO_BLOCKING_SORT, leads to its exclusion. + // NO_UNCOVERED_PROJECTIONS, leads to its exclusion. auto soln = buildWholeIXSoln(relevantIndices.front(), query, params); if (soln) { LOG(5) << "Planner: outputting soln that uses hinted index as scan."; diff --git a/src/mongo/db/query/query_planner_index_test.cpp b/src/mongo/db/query/query_planner_index_test.cpp index e56222e9657..ea8b176f302 100644 --- a/src/mongo/db/query/query_planner_index_test.cpp +++ b/src/mongo/db/query/query_planner_index_test.cpp @@ -585,20 +585,6 @@ TEST_F(QueryPlannerTest, CompoundMultikeyBoundsNoIntersect) { // QueryPlannerParams option tests // -TEST_F(QueryPlannerTest, NoBlockingSortsAllowedTest) { - params.options = QueryPlannerParams::NO_BLOCKING_SORT; - runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); - assertNumSolutions(0U); - - addIndex(BSON("x" << 1)); - - runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); - assertNumSolutions(1U); - assertSolutionExists( - "{fetch: {filter: null, node: {ixscan: " - "{filter: null, pattern: {x: 1}}}}}"); -} - TEST_F(QueryPlannerTest, NoTableScanBasic) { params.options = QueryPlannerParams::NO_TABLE_SCAN; runQuery(BSONObj()); diff --git a/src/mongo/db/query/query_planner_options_test.cpp b/src/mongo/db/query/query_planner_options_test.cpp index d2cdd50ea78..ef4070a70d8 100644 --- a/src/mongo/db/query/query_planner_options_test.cpp +++ b/src/mongo/db/query/query_planner_options_test.cpp @@ -351,15 +351,6 @@ TEST_F(QueryPlannerTest, HintInvalid) { runInvalidQueryHint(BSONObj(), fromjson("{b: 1}")); } -TEST_F(QueryPlannerTest, HintedBlockingSortIndexFilteredOut) { - params.options = QueryPlannerParams::NO_BLOCKING_SORT; - addIndex(BSON("a" << 1)); - addIndex(BSON("b" << 1)); - runQueryAsCommand( - fromjson("{find: 'testns', filter: {a: 1, b: 1}, sort: {b: 1}, hint: {a: 1}}")); - assertNumSolutions(0U); -} - TEST_F(QueryPlannerTest, HintedNotCoveredProjectionIndexFilteredOut) { params.options = QueryPlannerParams::NO_UNCOVERED_PROJECTIONS; addIndex(BSON("a" << 1)); diff --git a/src/mongo/db/query/query_planner_params.h b/src/mongo/db/query/query_planner_params.h index 9229dbb06ce..6597211ad7d 100644 --- a/src/mongo/db/query/query_planner_params.h +++ b/src/mongo/db/query/query_planner_params.h @@ -65,39 +65,36 @@ struct QueryPlannerParams { // See the comment on ShardFilterStage for details. INCLUDE_SHARD_FILTER = 1 << 2, - // Set this if you don't want any plans with a blocking sort stage. All sorts must be - // provided by an index. - NO_BLOCKING_SORT = 1 << 3, - // Set this if you want to turn on index intersection. - INDEX_INTERSECTION = 1 << 4, + INDEX_INTERSECTION = 1 << 3, - // Indicate to the planner that the caller is requesting a count operation, possibly through - // a count command, or as part of an aggregation pipeline. - IS_COUNT = 1 << 5, + // Indicate to the planner that this query could be eligible for count optimization. For + // example, the query {$group: {_id: null, sum: {$sum: 1}}} is a count-like operation and + // could be eligible for the COUNT_SCAN. + IS_COUNT = 1 << 4, // Set this if you want to handle batchSize properly with sort(). If limits on SORT // stages are always actually limits, then this should be left off. If they are // sometimes to be interpreted as batchSize, then this should be turned on. - SPLIT_LIMITED_SORT = 1 << 6, + SPLIT_LIMITED_SORT = 1 << 5, // Set this if you don't want any plans with a non-covered projection stage. All projections // must be provided/covered by an index. - NO_UNCOVERED_PROJECTIONS = 1 << 7, + NO_UNCOVERED_PROJECTIONS = 1 << 6, // Set this to generate covered whole IXSCAN plans. - GENERATE_COVERED_IXSCANS = 1 << 8, + GENERATE_COVERED_IXSCANS = 1 << 7, // Set this to track the most recent timestamp seen by this cursor while scanning the oplog. - TRACK_LATEST_OPLOG_TS = 1 << 9, + TRACK_LATEST_OPLOG_TS = 1 << 8, // Set this so that collection scans on the oplog wait for visibility before reading. - OPLOG_SCAN_WAIT_FOR_VISIBLE = 1 << 10, + OPLOG_SCAN_WAIT_FOR_VISIBLE = 1 << 9, // Set this so that getExecutorDistinct() will only use a plan that _guarantees_ it will // return exactly one document per value of the distinct field. See the comments above the // declaration of getExecutorDistinct() for more detail. - STRICT_DISTINCT_ONLY = 1 << 11, + STRICT_DISTINCT_ONLY = 1 << 10, }; // See Options enum above. |