diff options
36 files changed, 247 insertions, 168 deletions
diff --git a/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml b/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml index c08125ad738..c86f4bae027 100644 --- a/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml +++ b/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml @@ -8,3 +8,6 @@ # Disable featureFlagRequireTenantID until all paths pass tenant id to TenantNamespace # and TenantDatabase constructors. - featureFlagRequireTenantID +# Disable featureFlagSBELookupPushdown until integration tests can pass with basic SBE +# lookup translation. +- featureFlagSBELookupPushdown diff --git a/jstests/noPassthrough/lookup_pushdown.js b/jstests/noPassthrough/lookup_pushdown.js index 6d381d422af..acb2c51327e 100644 --- a/jstests/noPassthrough/lookup_pushdown.js +++ b/jstests/noPassthrough/lookup_pushdown.js @@ -1,5 +1,7 @@ /** * Tests basic functionality of pushing $lookup into the find layer. + * + * @tags: [requires_sharding] */ (function() { "use strict"; @@ -13,10 +15,11 @@ const JoinAlgorithm = { }; // Standalone cases. -const conn = MongoRunner.runMongod({setParameter: "internalEnableMultipleAutoGetCollections=true"}); +const conn = MongoRunner.runMongod({setParameter: "featureFlagSBELookupPushdown=true"}); assert.neq(null, conn, "mongod was unable to start up"); const name = "lookup_pushdown"; -let db = conn.getDB(name); +const foreignCollName = "foreign_lookup_pushdown"; +const viewName = "view_lookup_pushdown"; function runTest(coll, pipeline, expectedCode, aggOptions = {}, errMsgRegex = null) { const options = Object.assign({pipeline, cursor: {}}, aggOptions); @@ -34,14 +37,14 @@ function runTest(coll, pipeline, expectedCode, aggOptions = {}, errMsgRegex = nu } } +let db = conn.getDB(name); if (!checkSBEEnabled(db, ["featureFlagSBELookupPushdown"])) { - jsTestLog("Skipping test because the sbe lookup pushdown feature flag is disabled"); + jsTestLog("Skipping test because either the sbe lookup pushdown feature flag is disabled or" + + " sbe itself is disabled"); MongoRunner.stopMongod(conn); return; } -const foreignCollName = "foreign_lookup_pushdown"; -const viewName = "view_lookup_pushdown"; let coll = db[name]; assert.commandWorked(coll.insert({_id: 1, a: 2})); let foreignColl = db[foreignCollName]; @@ -54,6 +57,11 @@ runTest(coll, [{$lookup: {from: foreignCollName, localField: "a", foreignField: "b", as: "out"}}], JoinAlgorithm.NLJ /* expectedCode */); +// $lookup against a non-existent foreign collection. This $lookup is expected to be pushed down. +runTest(coll, + [{$lookup: {from: "nonexistentColl", localField: "a", foreignField: "b", as: "out"}}], + JoinAlgorithm.NLJ /* expectedCode */); + // Self join $lookup, no views. runTest(coll, [{$lookup: {from: name, localField: "a", foreignField: "a", as: "out"}}], @@ -108,7 +116,7 @@ runTest(coll, ], JoinAlgorithm.NLJ /* expectedCode */); -// Consecutive $lookups (first is against view). +// Consecutive $lookups, where the first $lookup is against a view. runTest(coll, [ {$lookup: {from: viewName, localField: "a", foreignField: "b", as: "out"}}, @@ -116,13 +124,15 @@ runTest(coll, ], false /* expectedCode */); -// Consecutive $lookups (first is against regular collection). +// Consecutive $lookups, where the first $lookup is against a regular collection. Here, neither +// $lookup is eligible for pushdown because currently, we can only know whether any secondary +// collection is a view or a sharded collection. runTest(coll, [ {$lookup: {from: foreignCollName, localField: "a", foreignField: "b", as: "out"}}, {$lookup: {from: viewName, localField: "a", foreignField: "b", as: "out"}} ], - JoinAlgorithm.NLJ /* expectedCode */); + false /* expectedCode */); // $lookup with pipeline. runTest(coll, @@ -280,7 +290,7 @@ MongoRunner.stopMongod(conn); const st = new ShardingTest({ shards: 2, mongos: 1, - other: {shardOptions: {setParameter: "internalEnableMultipleAutoGetCollections=true"}} + other: {shardOptions: {setParameter: "featureFlagSBELookupPushdown=true"}} }); db = st.s.getDB(name); diff --git a/jstests/noPassthrough/query_knobs_validation.js b/jstests/noPassthrough/query_knobs_validation.js index 8ff22a20308..6f21abc09ad 100644 --- a/jstests/noPassthrough/query_knobs_validation.js +++ b/jstests/noPassthrough/query_knobs_validation.js @@ -50,7 +50,6 @@ const expectedParamDefaults = { internalQueryIgnoreUnknownJSONSchemaKeywords: false, internalQueryProhibitBlockingMergeOnMongoS: false, internalQuerySlotBasedExecutionMaxStaticIndexScanIntervals: 1000, - internalEnableMultipleAutoGetCollections: false }; function assertDefaultParameterValues() { @@ -210,8 +209,5 @@ assertSetParameterFails("internalQuerySlotBasedExecutionMaxStaticIndexScanInterv assertSetParameterSucceeds("internalQueryForceClassicEngine", true); assertSetParameterSucceeds("internalQueryForceClassicEngine", false); -assertSetParameterSucceeds("internalEnableMultipleAutoGetCollections", true); -assertSetParameterSucceeds("internalEnableMultipleAutoGetCollections", false); - MongoRunner.stopMongod(conn); })(); diff --git a/src/mongo/db/commands/cqf/cqf_aggregate.cpp b/src/mongo/db/commands/cqf/cqf_aggregate.cpp index f092233de11..01d05a99293 100644 --- a/src/mongo/db/commands/cqf/cqf_aggregate.cpp +++ b/src/mongo/db/commands/cqf/cqf_aggregate.cpp @@ -269,7 +269,7 @@ static std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> optimizeAndCreateExe nullptr /*solution*/, {std::move(sbePlan), std::move(data)}, std::make_unique<ABTPrinter>(std::move(abtTree), phaseManager.getNodeToGroupPropsMap()), - &collection, + MultipleCollectionAccessor(collection), QueryPlannerParams::Options::DEFAULT, nss, std::move(yieldPolicy))); diff --git a/src/mongo/db/commands/run_aggregate.cpp b/src/mongo/db/commands/run_aggregate.cpp index 03a4edfeb62..813ad0e0a3e 100644 --- a/src/mongo/db/commands/run_aggregate.cpp +++ b/src/mongo/db/commands/run_aggregate.cpp @@ -531,7 +531,7 @@ std::vector<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> createLegacyEx std::unique_ptr<Pipeline, PipelineDeleter> pipeline, const LiteParsedPipeline& liteParsedPipeline, const NamespaceString& nss, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const AggregateCommandRequest& request, CurOp* curOp, const std::function<void(void)>& resetContextFn) { @@ -622,12 +622,17 @@ Status runAggregate(OperationContext* opCtx, // For operations on views, this will be the underlying namespace. NamespaceString nss = request.getNamespace(); - stdx::unordered_set<NamespaceString> secondaryExecNssList; // Determine if this aggregation has foreign collections that the execution subsystem needs // to be aware of. - if (internalEnableMultipleAutoGetCollections.load()) { - liteParsedPipeline.getForeignExecutionNamespaces(secondaryExecNssList); + std::vector<NamespaceStringOrUUID> secondaryExecNssList; + + // Taking locks over multiple collections is not supported in a transaction, nor is it + // supported outside of $lookup pushdown. + // TODO SERVER-64038: Remove this clause once MODE_IX multi-collection locking is supported. + if (!opCtx->inMultiDocumentTransaction() && + feature_flags::gFeatureFlagSBELookupPushdown.isEnabledAndIgnoreFCV()) { + secondaryExecNssList = liteParsedPipeline.getForeignExecutionNamespaces(); } // The collation to use for this aggregation. boost::optional to distinguish between the case @@ -642,30 +647,24 @@ Status runAggregate(OperationContext* opCtx, // connection is out of date. If the namespace is a view, the lock will be released before // re-running the expanded aggregation. boost::optional<AutoGetCollectionForReadCommandMaybeLockFree> ctx; - - // Vector of AutoGets for secondary collections. At the moment, this is internal to testing - // only because eventually, this will be replaced by 'AutoGetCollectionMulti'. - // TODO SERVER-62798: Replace this and the above AutoGet with 'AutoGetCollectionMulti'. - std::vector<std::unique_ptr<AutoGetCollectionForReadCommandMaybeLockFree>> secondaryCtx; - MultiCollection collections; + MultipleCollectionAccessor collections; auto initContext = [&](AutoGetCollectionViewMode m) -> void { - ctx.emplace(opCtx, nss, m); - for (const auto& ns : secondaryExecNssList) { - // Avoid locking the main namespace multiple times (we can't lock a secondary - // namespace multiple times because 'secondaryExecNssList is a set already). This - // emulates the behavior of 'AutoGetCollectionMulti'. - if (ns != nss) { - secondaryCtx.emplace_back( - std::make_unique<AutoGetCollectionForReadCommandMaybeLockFree>(opCtx, ns, m)); - } - } - collections = MultiCollection(ctx, secondaryCtx); + ctx.emplace(opCtx, + nss, + m, + Date_t::max(), + AutoStatsTracker::LogMode::kUpdateTopAndCurOp, + secondaryExecNssList); + collections = MultipleCollectionAccessor(opCtx, + &ctx->getCollection(), + ctx->getNss(), + ctx->isAnySecondaryNamespaceAViewOrSharded(), + secondaryExecNssList); }; auto resetContext = [&]() -> void { ctx.reset(); - secondaryCtx.clear(); collections.clear(); }; @@ -742,10 +741,9 @@ Status runAggregate(OperationContext* opCtx, opCtx, request.getCollation().get_value_or(BSONObj()), nullptr); collatorToUse.emplace(std::move(collator)); collatorToUseMatchesDefault = match; - tassert(6235101, "A collection-less aggregate should not take any locks", !ctx); - tassert(6235102, - "A collection-less aggregate should not take any secondary locks", - secondaryCtx.empty()); + tassert(6235101, + "A collection-less aggregate should not take any locks", + ctx == boost::none); } else { // This is a regular aggregation. Lock the collection or view. initContext(AutoGetCollectionViewMode::kViewsPermitted); diff --git a/src/mongo/db/db_raii.cpp b/src/mongo/db/db_raii.cpp index ab1f8220acf..116fc53a7e6 100644 --- a/src/mongo/db/db_raii.cpp +++ b/src/mongo/db/db_raii.cpp @@ -898,6 +898,11 @@ const NamespaceString& AutoGetCollectionForReadCommandMaybeLockFree::getNss() co } } +bool AutoGetCollectionForReadCommandMaybeLockFree::isAnySecondaryNamespaceAViewOrSharded() const { + return _autoGet ? _autoGet->isAnySecondaryNamespaceAViewOrSharded() + : _autoGetLockFree->isAnySecondaryNamespaceAViewOrSharded(); +} + AutoReadLockFree::AutoReadLockFree(OperationContext* opCtx, Date_t deadline) : _catalogStash(opCtx), _lockFreeReadsBlock(opCtx), diff --git a/src/mongo/db/db_raii.h b/src/mongo/db/db_raii.h index 0bc8a1058f9..a0aa871a428 100644 --- a/src/mongo/db/db_raii.h +++ b/src/mongo/db/db_raii.h @@ -438,6 +438,7 @@ public: const CollectionPtr& getCollection() const; const ViewDefinition* getView() const; const NamespaceString& getNss() const; + bool isAnySecondaryNamespaceAViewOrSharded() const; private: boost::optional<AutoGetCollectionForReadCommand> _autoGet; diff --git a/src/mongo/db/exec/sbe_cmd.cpp b/src/mongo/db/exec/sbe_cmd.cpp index c76885bec0d..bba9d728375 100644 --- a/src/mongo/db/exec/sbe_cmd.cpp +++ b/src/mongo/db/exec/sbe_cmd.cpp @@ -120,15 +120,16 @@ public: } root->attachToOperationContext(opCtx); - exec = uassertStatusOK(plan_executor_factory::make(opCtx, - std::move(cq), - nullptr, - {std::move(root), std::move(data)}, - {}, - &CollectionPtr::null, - false, /* returnOwnedBson */ - nss, - nullptr)); + exec = uassertStatusOK( + plan_executor_factory::make(opCtx, + std::move(cq), + nullptr, + {std::move(root), std::move(data)}, + {}, + MultipleCollectionAccessor(CollectionPtr::null), + false, /* returnOwnedBson */ + nss, + nullptr)); for (long long objCount = 0; objCount < batchSize; objCount++) { BSONObj next; PlanExecutor::ExecState state = exec->getNext(&next, nullptr); diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp index 184659a45bc..f5bc8dd7395 100644 --- a/src/mongo/db/pipeline/document_source_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_cursor.cpp @@ -137,12 +137,15 @@ void DocumentSourceCursor::loadBatch() { PlanExecutor::ExecState state; Document resultObj; - // TODO SERVER-62798: Replace this with 'AutoGetCollectionMulti'. boost::optional<AutoGetCollectionForReadMaybeLockFree> autoColl; tassert(5565800, "Expected PlanExecutor to use an external lock policy", _exec->lockPolicy() == PlanExecutor::LockPolicy::kLockExternally); - autoColl.emplace(pExpCtx->opCtx, _exec->nss()); + autoColl.emplace(pExpCtx->opCtx, + _exec->nss(), + AutoGetCollectionViewMode::kViewsForbidden, + Date_t::max(), + _exec->getSecondaryNamespaces()); uassertStatusOK(repl::ReplicationCoordinator::get(pExpCtx->opCtx) ->checkCanServeReadsFor(pExpCtx->opCtx, _exec->nss(), true)); diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp index 2f3cab580a5..57070e94da3 100644 --- a/src/mongo/db/pipeline/document_source_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_lookup.cpp @@ -1265,8 +1265,9 @@ intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( pExpCtx); // $lookup stages with local/foreignField specified are eligible for pushdown into SBE if - // the context allows it. - lookupStage->_sbeCompatible = pExpCtx->sbeCompatible; + // the context allows it and 'fromNs' does not correspond to a view. + lookupStage->_sbeCompatible = pExpCtx->sbeCompatible && + pExpCtx->getResolvedNamespace(lookupStage->_fromNs).pipeline.empty(); return lookupStage; } } diff --git a/src/mongo/db/pipeline/lite_parsed_pipeline.h b/src/mongo/db/pipeline/lite_parsed_pipeline.h index 35b401e3bdd..712d23c32bd 100644 --- a/src/mongo/db/pipeline/lite_parsed_pipeline.h +++ b/src/mongo/db/pipeline/lite_parsed_pipeline.h @@ -77,8 +77,8 @@ public: } /** - * Inserts the foreign collections(s) referenced by this stage that potentially will be involved - * in query execution, if any, into 'nssSet'. For example, consider the pipeline: + * Returns a vector of the foreign collections(s) referenced by this stage that potentially will + * be involved in query execution, if any. For example, consider the pipeline: * * [{$lookup: {from: "bar", localField: "a", foreignField: "b", as: "output"}}, * {$unionWith: {coll: "foo", pipeline: [...]}}]. @@ -87,10 +87,12 @@ public: * pushed down into the execution subsystem underneath the leading cursor stage, while "bar" * is considered one because "$lookup" can be pushed down in certain cases. */ - void getForeignExecutionNamespaces(stdx::unordered_set<NamespaceString>& nssSet) const { + std::vector<NamespaceStringOrUUID> getForeignExecutionNamespaces() const { + stdx::unordered_set<NamespaceString> nssSet; for (auto&& spec : _stageSpecs) { spec->getForeignExecutionNamespaces(nssSet); } + return {nssSet.begin(), nssSet.end()}; } /** diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp index f594ad1a3ea..445fd980d3b 100644 --- a/src/mongo/db/pipeline/pipeline_d.cpp +++ b/src/mongo/db/pipeline/pipeline_d.cpp @@ -114,13 +114,12 @@ namespace { * Lookup stages are extracted from the pipeline when all of the following conditions are met: * 0. When the 'internalQueryForceClassicEngine' feature flag is 'false'. * 1. When the 'featureFlagSBELookupPushdown' feature flag is 'true'. - * 2. When the 'internalEnableMultipleAutoGetCollections' flag is 'true' - * 3. The $lookup uses only the 'localField'/'foreignField' syntax (no pipelines). - * 4. The foreign collection is neither sharded nor a view. + * 2. The $lookup uses only the 'localField'/'foreignField' syntax (no pipelines). + * 3. The foreign collection is neither sharded nor a view. */ std::vector<std::unique_ptr<InnerPipelineStageInterface>> extractSbeCompatibleStagesForPushdown( const intrusive_ptr<ExpressionContext>& expCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery* cq, Pipeline* pipeline) { // We will eventually use the extracted group stages to populate 'CanonicalQuery::pipeline' @@ -139,12 +138,15 @@ std::vector<std::unique_ptr<InnerPipelineStageInterface>> extractSbeCompatibleSt auto&& sources = pipeline->getSources(); + const auto groupFeatureFlagEnabled = feature_flags::gFeatureFlagSBEGroupPushdown.isEnabled( + serverGlobalParams.featureCompatibility); + const auto lookupFeatureFlagEnabled = + feature_flags::gFeatureFlagSBELookupPushdown.isEnabledAndIgnoreFCV(); for (auto itr = sources.begin(); itr != sources.end();) { // $group pushdown logic. if (auto groupStage = dynamic_cast<DocumentSourceGroup*>(itr->get())) { - bool groupEligibleForPushdown = feature_flags::gFeatureFlagSBEGroupPushdown.isEnabled( - serverGlobalParams.featureCompatibility) && - groupStage->sbeCompatible() && !groupStage->doingMerge(); + bool groupEligibleForPushdown = + groupFeatureFlagEnabled && groupStage->sbeCompatible() && !groupStage->doingMerge(); if (groupEligibleForPushdown) { stagesForPushdown.push_back(std::make_unique<InnerPipelineStageImpl>(groupStage)); sources.erase(itr++); @@ -155,22 +157,30 @@ std::vector<std::unique_ptr<InnerPipelineStageInterface>> extractSbeCompatibleSt // $lookup pushdown logic. if (auto lookupStage = dynamic_cast<DocumentSourceLookUp*>(itr->get())) { + // If lookup pushdown isn't enabled, then neither this stage nor any subsequent stages + // will be eligible for pushdown. As such, we early return to avoid unnecessary work. + if (!lookupFeatureFlagEnabled) { + break; + } + bool isForeignSharded = false; - bool isForeignView = false; const auto& fromNs = lookupStage->getFromNs(); const auto& foreignColl = collections.lookupCollection(fromNs); if (foreignColl) { isForeignSharded = foreignColl.isSharded(); - } else { - // If the right hand side targets a namespace that we can't find in - // 'collections', we infer that it is targeting a view. - isForeignView = true; } - bool lookupEligibleForPushdown = - feature_flags::gFeatureFlagSBELookupPushdown.isEnabledAndIgnoreFCV() && - internalEnableMultipleAutoGetCollections.load() && lookupStage->sbeCompatible() && - !isForeignSharded && !isForeignView; + // When acquiring locks for multiple collections, it is the case that we can only + // determine whether any secondary collection is a view or is sharded, not which ones + // are a view or are sharded and which ones aren't. As such, if any secondary collection + // is a view or is sharded, no $lookup will be eligible for pushdown. + // Note that we still check 'isForeignSharded' because this flag will be accurate in + // the event that the main collection is a secondary collection. + // Also note that 'lookupStage->sbeCompatible()' encodes whether the foreign + // collection is a view. + bool lookupEligibleForPushdown = lookupFeatureFlagEnabled && + lookupStage->sbeCompatible() && !isForeignSharded && + !collections.isAnySecondaryNamespaceAViewOrSharded(); if (lookupEligibleForPushdown) { stagesForPushdown.push_back(std::make_unique<InnerPipelineStageImpl>(lookupStage)); sources.erase(itr++); @@ -188,7 +198,7 @@ std::vector<std::unique_ptr<InnerPipelineStageInterface>> extractSbeCompatibleSt StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> attemptToGetExecutor( const intrusive_ptr<ExpressionContext>& expCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const NamespaceString& nss, BSONObj queryObj, BSONObj projectionObj, @@ -662,7 +672,7 @@ PipelineD::buildInnerQueryExecutorSample(DocumentSourceSample* sampleStage, } std::pair<PipelineD::AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> -PipelineD::buildInnerQueryExecutor(const MultiCollection& collections, +PipelineD::buildInnerQueryExecutor(const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline) { @@ -703,7 +713,7 @@ PipelineD::buildInnerQueryExecutor(const MultiCollection& collections, } void PipelineD::attachInnerQueryExecutorToPipeline( - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, PipelineD::AttachExecutorCallback attachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, Pipeline* pipeline) { @@ -717,7 +727,7 @@ void PipelineD::attachInnerQueryExecutorToPipeline( } void PipelineD::buildAndAttachInnerQueryExecutorToPipeline( - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline) { @@ -853,7 +863,7 @@ auto buildProjectionForPushdown(const DepsTracker& deps, } // namespace std::pair<PipelineD::AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> -PipelineD::buildInnerQueryExecutorGeneric(const MultiCollection& collections, +PipelineD::buildInnerQueryExecutorGeneric(const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline) { @@ -944,7 +954,7 @@ PipelineD::buildInnerQueryExecutorGeneric(const MultiCollection& collections, } std::pair<PipelineD::AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> -PipelineD::buildInnerQueryExecutorGeoNear(const MultiCollection& collections, +PipelineD::buildInnerQueryExecutorGeoNear(const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline) { @@ -1007,7 +1017,7 @@ PipelineD::buildInnerQueryExecutorGeoNear(const MultiCollection& collections, StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prepareExecutor( const intrusive_ptr<ExpressionContext>& expCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const NamespaceString& nss, Pipeline* pipeline, const boost::intrusive_ptr<DocumentSourceSort>& sortStage, diff --git a/src/mongo/db/pipeline/pipeline_d.h b/src/mongo/db/pipeline/pipeline_d.h index 340ce818ccd..fb69cce46f9 100644 --- a/src/mongo/db/pipeline/pipeline_d.h +++ b/src/mongo/db/pipeline/pipeline_d.h @@ -41,7 +41,7 @@ #include "mongo/db/pipeline/document_source_internal_unpack_bucket.h" #include "mongo/db/pipeline/document_source_sample.h" #include "mongo/db/query/collation/collator_factory_interface.h" -#include "mongo/db/query/multi_collection.h" +#include "mongo/db/query/multiple_collection_accessor.h" #include "mongo/db/query/plan_executor.h" namespace mongo { @@ -94,7 +94,7 @@ public: * 'nullptr'. */ static std::pair<AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> - buildInnerQueryExecutor(const MultiCollection& collections, + buildInnerQueryExecutor(const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline); @@ -107,7 +107,7 @@ public: * collections. */ static void attachInnerQueryExecutorToPipeline( - const MultiCollection& collection, + const MultipleCollectionAccessor& collection, AttachExecutorCallback attachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, Pipeline* pipeline); @@ -119,7 +119,7 @@ public: * can be created right after building the executor. */ static void buildAndAttachInnerQueryExecutorToPipeline( - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline); @@ -168,7 +168,7 @@ private: * the 'pipeline'. */ static std::pair<AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> - buildInnerQueryExecutorGeneric(const MultiCollection& collections, + buildInnerQueryExecutorGeneric(const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline); @@ -177,8 +177,8 @@ private: * Creates a PlanExecutor to be used in the initial cursor source. This function will try to * push down the $sort, $project, $match and $limit stages into the PlanStage layer whenever * possible. In this case, these stages will be incorporated into the PlanExecutor. Note that - * this function takes a 'MultiCollection' because certain $lookup stages that reference - * multiple collections may be eligible for pushdown in the PlanExecutor. + * this function takes a 'MultipleCollectionAccessor' because certain $lookup stages that + * reference multiple collections may be eligible for pushdown in the PlanExecutor. * * Set 'rewrittenGroupStage' when the pipeline uses $match+$sort+$group stages that are * compatible with a DISTINCT_SCAN plan that visits the first document in each group @@ -189,7 +189,7 @@ private: */ static StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> prepareExecutor( const boost::intrusive_ptr<ExpressionContext>& expCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const NamespaceString& nss, Pipeline* pipeline, const boost::intrusive_ptr<DocumentSourceSort>& sortStage, @@ -207,12 +207,12 @@ private: * defined on 'collections' does not exist, as the $geoNearCursor requires a 2d or 2dsphere * index. * - * Note that this method takes a 'MultiCollection' even though DocumentSourceGeoNearCursor - * only operates over a single collection because the underlying execution API expects a - * 'MultiCollection'. + * Note that this method takes a 'MultipleCollectionAccessor' even though + * DocumentSourceGeoNearCursor only operates over a single collection because the underlying + * execution API expects a 'MultipleCollectionAccessor'. */ static std::pair<AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> - buildInnerQueryExecutorGeoNear(const MultiCollection& collections, + buildInnerQueryExecutorGeoNear(const MultipleCollectionAccessor& collections, const NamespaceString& nss, const AggregateCommandRequest* aggRequest, Pipeline* pipeline); diff --git a/src/mongo/db/pipeline/plan_executor_pipeline.h b/src/mongo/db/pipeline/plan_executor_pipeline.h index 66313dd22c5..efef0a7bb2d 100644 --- a/src/mongo/db/pipeline/plan_executor_pipeline.h +++ b/src/mongo/db/pipeline/plan_executor_pipeline.h @@ -64,6 +64,15 @@ public: return _expCtx->ns; } + const std::vector<NamespaceStringOrUUID>& getSecondaryNamespaces() const final { + // Return a reference to an empty static array. This array will never contain any elements + // because even though a PlanExecutorPipeline can reference multiple collections, it never + // takes any locks over said namespaces (this is the responsibility of DocumentSources + // which internally manage their own PlanExecutors). + const static std::vector<NamespaceStringOrUUID> emptyNssVector; + return emptyNssVector; + } + OperationContext* getOpCtx() const override { return _expCtx->opCtx; } diff --git a/src/mongo/db/pipeline/process_interface/common_mongod_process_interface.cpp b/src/mongo/db/pipeline/process_interface/common_mongod_process_interface.cpp index 06177205d92..c8d74ddd550 100644 --- a/src/mongo/db/pipeline/process_interface/common_mongod_process_interface.cpp +++ b/src/mongo/db/pipeline/process_interface/common_mongod_process_interface.cpp @@ -441,7 +441,7 @@ CommonMongodProcessInterface::attachCursorSourceToPipelineForLocalRead(Pipeline* Date_t::max(), AutoStatsTracker::LogMode::kUpdateTop); - MultiCollection holder{autoColl->getCollection()}; + MultipleCollectionAccessor holder{autoColl->getCollection()}; PipelineD::buildAndAttachInnerQueryExecutorToPipeline( holder, expCtx->ns, nullptr, pipeline.get()); diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index 59528380bc8..750a8a3fe14 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -381,7 +381,9 @@ void fillOutPlannerParams(OperationContext* opCtx, } std::map<NamespaceString, SecondaryCollectionInfo> fillOutSecondaryCollectionsInformation( - OperationContext* opCtx, const MultiCollection& collections, CanonicalQuery* canonicalQuery) { + OperationContext* opCtx, + const MultipleCollectionAccessor& collections, + CanonicalQuery* canonicalQuery) { std::map<NamespaceString, SecondaryCollectionInfo> infoMap; bool apiStrict = APIParameters::get(opCtx).getAPIStrict().value_or(false); auto fillOutSecondaryInfo = [&](const NamespaceString& nss, @@ -410,7 +412,7 @@ std::map<NamespaceString, SecondaryCollectionInfo> fillOutSecondaryCollectionsIn } void fillOutPlannerParams(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, CanonicalQuery* canonicalQuery, QueryPlannerParams* plannerParams) { fillOutPlannerParams(opCtx, collections.getMainCollection(), canonicalQuery, plannerParams); @@ -983,7 +985,7 @@ public: using PrepareExecutionHelper::PrepareExecutionHelper; SlotBasedPrepareExecutionHelper(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, CanonicalQuery* cq, PlanYieldPolicy* yieldPolicy, size_t plannerOptions) @@ -1179,7 +1181,7 @@ protected: } private: - const MultiCollection& _collections; + const MultipleCollectionAccessor& _collections; }; StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getClassicExecutor( @@ -1222,7 +1224,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getClassicExecu */ std::unique_ptr<sbe::RuntimePlanner> makeRuntimePlannerIfNeeded( OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, CanonicalQuery* canonicalQuery, size_t numSolutions, boost::optional<size_t> decisionWorks, @@ -1293,7 +1295,7 @@ std::unique_ptr<PlanYieldPolicySBE> makeSbeYieldPolicy( StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExecutor( OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, std::unique_ptr<CanonicalQuery> cq, std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, PlanYieldPolicy::YieldPolicy requestedYieldPolicy, @@ -1338,7 +1340,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExe return plan_executor_factory::make(opCtx, std::move(cq), std::move(candidates), - mainColl, + collections, plannerOptions, std::move(nss), std::move(yieldPolicy)); @@ -1364,7 +1366,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExe std::move(solutions[0]), std::move(roots[0]), {}, - mainColl, + collections, plannerOptions, std::move(nss), std::move(yieldPolicy)); @@ -1373,7 +1375,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExe StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor( OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, std::unique_ptr<CanonicalQuery> canonicalQuery, std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, PlanYieldPolicy::YieldPolicy yieldPolicy, @@ -1399,7 +1401,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor( std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, PlanYieldPolicy::YieldPolicy yieldPolicy, size_t plannerOptions) { - MultiCollection multi{collection}; + MultipleCollectionAccessor multi{collection}; return getExecutor(opCtx, multi, std::move(canonicalQuery), @@ -1414,7 +1416,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor( StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorFind( OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, std::unique_ptr<CanonicalQuery> canonicalQuery, std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, bool permitYield, @@ -1442,7 +1444,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorFind std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, bool permitYield, size_t plannerOptions) { - MultiCollection multi{*coll}; + MultipleCollectionAccessor multi{*coll}; return getExecutorFind(opCtx, multi, std::move(canonicalQuery), diff --git a/src/mongo/db/query/get_executor.h b/src/mongo/db/query/get_executor.h index 09dd1722412..47987a362c4 100644 --- a/src/mongo/db/query/get_executor.h +++ b/src/mongo/db/query/get_executor.h @@ -38,7 +38,7 @@ #include "mongo/db/ops/update_request.h" #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/count_command_gen.h" -#include "mongo/db/query/multi_collection.h" +#include "mongo/db/query/multiple_collection_accessor.h" #include "mongo/db/query/parsed_distinct.h" #include "mongo/db/query/plan_executor.h" #include "mongo/db/query/query_planner_params.h" @@ -85,7 +85,9 @@ void fillOutIndexEntries(OperationContext* opCtx, * Fills out information about secondary collections held by 'collections' in 'plannerParams'. */ std::map<NamespaceString, SecondaryCollectionInfo> fillOutSecondaryCollectionsInformation( - OperationContext* opCtx, const MultiCollection& collections, CanonicalQuery* canonicalQuery); + OperationContext* opCtx, + const MultipleCollectionAccessor& collections, + CanonicalQuery* canonicalQuery); /** * Fill out the provided 'plannerParams' for the 'canonicalQuery' operating on the collection @@ -103,7 +105,7 @@ void fillOutPlannerParams(OperationContext* opCtx, * secondary collections held by 'collections' on 'plannerParams'. */ void fillOutPlannerParams(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, CanonicalQuery* canonicalQuery, QueryPlannerParams* plannerParams); @@ -162,13 +164,13 @@ bool shouldWaitForOplogVisibility(OperationContext* opCtx, * attach them to the provided 'CanonicalQuery'. This function should capture the Pipeline that * stages should be extracted from. * - * Note that the first overload takes a 'MultiCollection' and can construct a PlanExecutor over - * multiple collections, while the second overload takes a single 'CollectionPtr' and can only - * construct a PlanExecutor over a single collection. + * Note that the first overload takes a 'MultipleCollectionAccessor' and can construct a + * PlanExecutor over multiple collections, while the second overload takes a single 'CollectionPtr' + * and can only construct a PlanExecutor over a single collection. */ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor( OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, std::unique_ptr<CanonicalQuery> canonicalQuery, std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, PlanYieldPolicy::YieldPolicy yieldPolicy, @@ -197,13 +199,13 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor( * attach them to the provided 'CanonicalQuery'. This function should capture the Pipeline that * stages should be extracted from. * - * Note that the first overload takes a 'MultiCollection' and can construct a PlanExecutor over - * multiple collections, while the second overload takes a single 'CollectionPtr' and can only - * construct a PlanExecutor over a single collection. + * Note that the first overload takes a 'MultipleCollectionAccessor' and can construct a + * PlanExecutor over multiple collections, while the second overload takes a single + * 'CollectionPtr' and can only construct a PlanExecutor over a single collection. */ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorFind( OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, std::unique_ptr<CanonicalQuery> canonicalQuery, std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, bool permitYield = false, diff --git a/src/mongo/db/query/multi_collection.h b/src/mongo/db/query/multiple_collection_accessor.h index 1e7a04357af..981e2d5c568 100644 --- a/src/mongo/db/query/multi_collection.h +++ b/src/mongo/db/query/multiple_collection_accessor.h @@ -31,41 +31,45 @@ #include "mongo/db/catalog/collection.h" #include "mongo/db/db_raii.h" -#include "mongo/db/query/query_planner_params.h" namespace mongo { /** * Class which holds a set of pointers to multiple collections. This class distinguishes between * a 'main collection' and 'secondary collections'. While the former represents the collection a - * given command is run against, the latter represents other collections that the execution - * engine may need to be made aware of. + * given command is run against, the latter represents other collections that the query execution + * engine may need to access. */ -class MultiCollection final { +class MultipleCollectionAccessor final { public: - MultiCollection() = default; + MultipleCollectionAccessor() = default; - MultiCollection(boost::optional<AutoGetCollectionForReadCommandMaybeLockFree>& mainCollCtx, - std::vector<std::unique_ptr<AutoGetCollectionForReadCommandMaybeLockFree>>& - secondaryCollCtxes) { - if (mainCollCtx) { - _mainColl = &mainCollCtx->getCollection(); - } + MultipleCollectionAccessor(OperationContext* opCtx, + const CollectionPtr* mainColl, + const NamespaceString& mainCollNss, + bool isAnySecondaryNamespaceAViewOrSharded, + const std::vector<NamespaceStringOrUUID>& secondaryExecNssList) + : _mainColl(mainColl), + _isAnySecondaryNamespaceAViewOrSharded(isAnySecondaryNamespaceAViewOrSharded) { + auto catalog = CollectionCatalog::get(opCtx); + for (const auto& secondaryNssOrUuid : secondaryExecNssList) { + auto secondaryNss = catalog->resolveNamespaceStringOrUUID(opCtx, secondaryNssOrUuid); - for (auto& secondaryColl : secondaryCollCtxes) { - if (*secondaryColl) { - // Even if 'secondaryColl' doesn't exist, we still want to include it. It is the - // responsibility of consumers of this class to verify that a collection exists - // before accessing it. - _secondaryColls.emplace(secondaryColl->getNss(), secondaryColl->getCollection()); + // Don't store a CollectionPtr if the main nss is also a secondary one. + if (secondaryNss != mainCollNss) { + // Even if the collection corresponding to 'secondaryNss' doesn't exist, we + // still want to include it. It is the responsibility of consumers of this class + // to verify that a collection exists before accessing it. + _secondaryColls.emplace(std::move(secondaryNss), + catalog->lookupCollectionByNamespace(opCtx, secondaryNss)); } } } - explicit MultiCollection(const CollectionPtr* mainColl) - : _mainColl(mainColl), _secondaryColls({}) {} + explicit MultipleCollectionAccessor(const CollectionPtr* mainColl) : _mainColl(mainColl) {} - explicit MultiCollection(const CollectionPtr& mainColl) : MultiCollection(&mainColl) {} + explicit MultipleCollectionAccessor(const CollectionPtr& mainColl) + : MultipleCollectionAccessor(&mainColl) {} bool hasMainCollection() const { return _mainColl->get(); @@ -75,10 +79,22 @@ public: return *_mainColl; } - const std::map<NamespaceString, const CollectionPtr&>& getSecondaryCollections() const { + const std::map<NamespaceString, CollectionPtr>& getSecondaryCollections() const { return _secondaryColls; } + std::vector<NamespaceStringOrUUID> getSecondaryCollectionVector() const { + std::vector<NamespaceStringOrUUID> secondaryCollectionList; + for (const auto& [nss, _] : _secondaryColls) { + secondaryCollectionList.emplace_back(nss); + } + return secondaryCollectionList; + } + + bool isAnySecondaryNamespaceAViewOrSharded() const { + return _isAnySecondaryNamespaceAViewOrSharded; + } + const CollectionPtr& lookupCollection(const NamespaceString& nss) const { if (_mainColl && nss == _mainColl->get()->ns()) { return *_mainColl; @@ -96,7 +112,13 @@ public: private: const CollectionPtr* _mainColl{&CollectionPtr::null}; + // Tracks whether any secondary namespace is a view or sharded based on information captured + // at the time of lock acquisition. This is used to determine if a $lookup is eligible for + // pushdown into the query execution subsystem as $lookup against a foreign view or a foreign + // sharded collection is not currently supported by the execution subsystem. + bool _isAnySecondaryNamespaceAViewOrSharded = false; + // Map from namespace to a corresponding CollectionPtr. - std::map<NamespaceString, const CollectionPtr&> _secondaryColls{}; + std::map<NamespaceString, CollectionPtr> _secondaryColls{}; }; } // namespace mongo diff --git a/src/mongo/db/query/plan_executor.h b/src/mongo/db/query/plan_executor.h index 1d8b70057b6..9748e3de303 100644 --- a/src/mongo/db/query/plan_executor.h +++ b/src/mongo/db/query/plan_executor.h @@ -170,6 +170,11 @@ public: virtual const NamespaceString& nss() const = 0; /** + * Returns a vector of secondary namespaces that are relevant to this executor. + */ + virtual const std::vector<NamespaceStringOrUUID>& getSecondaryNamespaces() const = 0; + + /** * Return the OperationContext that the plan is currently executing within. */ virtual OperationContext* getOpCtx() const = 0; diff --git a/src/mongo/db/query/plan_executor_factory.cpp b/src/mongo/db/query/plan_executor_factory.cpp index c72b768adb2..288b4a85571 100644 --- a/src/mongo/db/query/plan_executor_factory.cpp +++ b/src/mongo/db/query/plan_executor_factory.cpp @@ -123,11 +123,10 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( std::unique_ptr<QuerySolution> solution, std::pair<std::unique_ptr<sbe::PlanStage>, stage_builder::PlanStageData> root, std::unique_ptr<optimizer::AbstractABTPrinter> optimizerData, - const CollectionPtr* collection, + const MultipleCollectionAccessor& collections, size_t plannerOptions, NamespaceString nss, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy) { - dassert(collection); auto&& [rootStage, data] = root; LOGV2_DEBUG(4822860, @@ -145,9 +144,9 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( {makeVector<sbe::plan_ranker::CandidatePlan>(sbe::plan_ranker::CandidatePlan{ std::move(solution), std::move(rootStage), std::move(data)}), 0}, - *collection, plannerOptions & QueryPlannerParams::RETURN_OWNED_DATA, std::move(nss), + collections.getSecondaryCollectionVector(), false, std::move(yieldPolicy)), PlanExecutor::Deleter{opCtx}}}; @@ -157,11 +156,10 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( OperationContext* opCtx, std::unique_ptr<CanonicalQuery> cq, sbe::CandidatePlans candidates, - const CollectionPtr* collection, + const MultipleCollectionAccessor& collections, size_t plannerOptions, NamespaceString nss, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy) { - dassert(collection); LOGV2_DEBUG(4822861, 5, @@ -173,9 +171,9 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( std::move(cq), {}, std::move(candidates), - *collection, plannerOptions & QueryPlannerParams::RETURN_OWNED_DATA, std::move(nss), + collections.getSecondaryCollectionVector(), true, std::move(yieldPolicy)), PlanExecutor::Deleter{opCtx}}}; diff --git a/src/mongo/db/query/plan_executor_factory.h b/src/mongo/db/query/plan_executor_factory.h index 17694489add..bf41f169af9 100644 --- a/src/mongo/db/query/plan_executor_factory.h +++ b/src/mongo/db/query/plan_executor_factory.h @@ -115,7 +115,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( std::unique_ptr<QuerySolution> solution, std::pair<std::unique_ptr<sbe::PlanStage>, stage_builder::PlanStageData> root, std::unique_ptr<optimizer::AbstractABTPrinter> optimizerData, - const CollectionPtr* collection, + const MultipleCollectionAccessor& collections, size_t plannerOptions, NamespaceString nss, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy); @@ -129,7 +129,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( OperationContext* opCtx, std::unique_ptr<CanonicalQuery> cq, sbe::CandidatePlans candidates, - const CollectionPtr* collection, + const MultipleCollectionAccessor& collections, size_t plannerOptions, NamespaceString nss, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy); diff --git a/src/mongo/db/query/plan_executor_impl.cpp b/src/mongo/db/query/plan_executor_impl.cpp index 241b52ed58f..ba9b2a57922 100644 --- a/src/mongo/db/query/plan_executor_impl.cpp +++ b/src/mongo/db/query/plan_executor_impl.cpp @@ -230,6 +230,14 @@ const NamespaceString& PlanExecutorImpl::nss() const { return _nss; } +const std::vector<NamespaceStringOrUUID>& PlanExecutorImpl::getSecondaryNamespaces() const { + // Return a reference to an empty static array. This array will never contain any elements + // because a PlanExecutorImpl is only capable of executing against a single namespace. As + // such, it will never lock more than one namespace. + const static std::vector<NamespaceStringOrUUID> emptyNssVector; + return emptyNssVector; +} + OperationContext* PlanExecutorImpl::getOpCtx() const { return _opCtx; } diff --git a/src/mongo/db/query/plan_executor_impl.h b/src/mongo/db/query/plan_executor_impl.h index c4642dd777e..7f711c111cc 100644 --- a/src/mongo/db/query/plan_executor_impl.h +++ b/src/mongo/db/query/plan_executor_impl.h @@ -67,6 +67,7 @@ public: virtual ~PlanExecutorImpl(); CanonicalQuery* getCanonicalQuery() const final; const NamespaceString& nss() const final; + const std::vector<NamespaceStringOrUUID>& getSecondaryNamespaces() const final; OperationContext* getOpCtx() const final; void saveState() final; void restoreState(const RestoreContext& context) final; diff --git a/src/mongo/db/query/plan_executor_sbe.cpp b/src/mongo/db/query/plan_executor_sbe.cpp index e1dded76398..d2a29fa3a9d 100644 --- a/src/mongo/db/query/plan_executor_sbe.cpp +++ b/src/mongo/db/query/plan_executor_sbe.cpp @@ -50,14 +50,15 @@ PlanExecutorSBE::PlanExecutorSBE(OperationContext* opCtx, std::unique_ptr<CanonicalQuery> cq, std::unique_ptr<optimizer::AbstractABTPrinter> optimizerData, sbe::CandidatePlans candidates, - const CollectionPtr& collection, bool returnOwnedBson, NamespaceString nss, + std::vector<NamespaceStringOrUUID> secondaryNssVector, bool isOpen, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy) : _state{isOpen ? State::kOpened : State::kClosed}, _opCtx(opCtx), _nss(std::move(nss)), + _secondaryNssVector(std::move(secondaryNssVector)), _mustReturnOwnedBson(returnOwnedBson), _root{std::move(candidates.winner().root)}, _rootData{std::move(candidates.winner().data)}, diff --git a/src/mongo/db/query/plan_executor_sbe.h b/src/mongo/db/query/plan_executor_sbe.h index f20e399d36e..8e1d2f35ea5 100644 --- a/src/mongo/db/query/plan_executor_sbe.h +++ b/src/mongo/db/query/plan_executor_sbe.h @@ -47,9 +47,9 @@ public: std::unique_ptr<CanonicalQuery> cq, std::unique_ptr<optimizer::AbstractABTPrinter> optimizerData, sbe::CandidatePlans candidates, - const CollectionPtr& collection, bool returnOwnedBson, NamespaceString nss, + std::vector<NamespaceStringOrUUID> secondaryNssVector, bool isOpen, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy); @@ -61,6 +61,10 @@ public: return _nss; } + const std::vector<NamespaceStringOrUUID>& getSecondaryNamespaces() const final { + return _secondaryNssVector; + } + OperationContext* getOpCtx() const override { return _opCtx; } @@ -148,6 +152,9 @@ private: OperationContext* _opCtx; NamespaceString _nss; + + // Vector of secondary namespaces. + std::vector<NamespaceStringOrUUID> _secondaryNssVector{}; const bool _mustReturnOwnedBson; // CompileCtx owns the instance pointed by _env, so we must keep it around. diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index be7175e4d8b..32940555877 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -693,13 +693,6 @@ server_parameters: validator: gt: 0 - internalEnableMultipleAutoGetCollections: - description: "Test only parameter to enable taking multiple AutoGetCollections in runAggregate" - set_at: [ startup, runtime ] - cpp_varname: "internalEnableMultipleAutoGetCollections" - cpp_vartype: AtomicWord<bool> - default: false - internalQueryEnableSamplingCardinalityEstimator: description: "Set to use the sampling-based method for estimating cardinality in the Cascades optimizer." diff --git a/src/mongo/db/query/query_planner.h b/src/mongo/db/query/query_planner.h index 142a6f6d7c7..c5e07796585 100644 --- a/src/mongo/db/query/query_planner.h +++ b/src/mongo/db/query/query_planner.h @@ -32,7 +32,7 @@ #include "mongo/base/string_data.h" #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/classic_plan_cache.h" -#include "mongo/db/query/multi_collection.h" +#include "mongo/db/query/multiple_collection_accessor.h" #include "mongo/db/query/query_planner_params.h" #include "mongo/db/query/query_solution.h" diff --git a/src/mongo/db/query/sbe_cached_solution_planner.h b/src/mongo/db/query/sbe_cached_solution_planner.h index d297359f7a2..0fa3d1cbb14 100644 --- a/src/mongo/db/query/sbe_cached_solution_planner.h +++ b/src/mongo/db/query/sbe_cached_solution_planner.h @@ -45,7 +45,7 @@ namespace mongo::sbe { class CachedSolutionPlanner final : public BaseRuntimePlanner { public: CachedSolutionPlanner(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QueryPlannerParams& queryParams, size_t decisionReads, diff --git a/src/mongo/db/query/sbe_multi_planner.h b/src/mongo/db/query/sbe_multi_planner.h index cae5a027f1c..976c54581f3 100644 --- a/src/mongo/db/query/sbe_multi_planner.h +++ b/src/mongo/db/query/sbe_multi_planner.h @@ -43,7 +43,7 @@ namespace mongo::sbe { class MultiPlanner final : public BaseRuntimePlanner { public: MultiPlanner(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QueryPlannerParams& qpp, PlanCachingMode cachingMode, diff --git a/src/mongo/db/query/sbe_runtime_planner.h b/src/mongo/db/query/sbe_runtime_planner.h index eef0a59acbc..c360105c01c 100644 --- a/src/mongo/db/query/sbe_runtime_planner.h +++ b/src/mongo/db/query/sbe_runtime_planner.h @@ -33,8 +33,9 @@ #include "mongo/db/exec/sbe/stages/stages.h" #include "mongo/db/query/all_indices_required_checker.h" #include "mongo/db/query/canonical_query.h" -#include "mongo/db/query/multi_collection.h" +#include "mongo/db/query/multiple_collection_accessor.h" #include "mongo/db/query/plan_yield_policy_sbe.h" +#include "mongo/db/query/query_planner_params.h" #include "mongo/db/query/query_solution.h" #include "mongo/db/query/sbe_plan_ranker.h" @@ -77,7 +78,7 @@ public: class BaseRuntimePlanner : public RuntimePlanner { public: BaseRuntimePlanner(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QueryPlannerParams& queryParams, PlanYieldPolicySBE* yieldPolicy) @@ -133,7 +134,7 @@ protected: size_t maxTrialPeriodNumReads); OperationContext* const _opCtx; - const MultiCollection& _collections; + const MultipleCollectionAccessor& _collections; const CanonicalQuery& _cq; const QueryPlannerParams _queryParams; PlanYieldPolicySBE* const _yieldPolicy; diff --git a/src/mongo/db/query/sbe_stage_builder.cpp b/src/mongo/db/query/sbe_stage_builder.cpp index 09305310058..7324b378d58 100644 --- a/src/mongo/db/query/sbe_stage_builder.cpp +++ b/src/mongo/db/query/sbe_stage_builder.cpp @@ -645,7 +645,7 @@ std::unique_ptr<fts::FTSMatcher> makeFtsMatcher(OperationContext* opCtx, } // namespace SlotBasedStageBuilder::SlotBasedStageBuilder(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QuerySolution& solution, PlanYieldPolicySBE* yieldPolicy, diff --git a/src/mongo/db/query/sbe_stage_builder.h b/src/mongo/db/query/sbe_stage_builder.h index 8d5d14f20e8..5ab26bb9414 100644 --- a/src/mongo/db/query/sbe_stage_builder.h +++ b/src/mongo/db/query/sbe_stage_builder.h @@ -34,7 +34,7 @@ #include "mongo/db/exec/sbe/values/slot.h" #include "mongo/db/exec/sbe/values/value.h" #include "mongo/db/exec/trial_period_utils.h" -#include "mongo/db/query/multi_collection.h" +#include "mongo/db/query/multiple_collection_accessor.h" #include "mongo/db/query/plan_yield_policy_sbe.h" #include "mongo/db/query/sbe_stage_builder_helpers.h" #include "mongo/db/query/shard_filterer_factory_interface.h" @@ -325,7 +325,7 @@ public: static constexpr StringData kIndexKeyPattern = PlanStageSlots::kIndexKeyPattern; SlotBasedStageBuilder(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QuerySolution& solution, PlanYieldPolicySBE* yieldPolicy, @@ -440,7 +440,7 @@ private: sbe::value::FrameIdGenerator _frameIdGenerator; sbe::value::SpoolIdGenerator _spoolIdGenerator; - const MultiCollection& _collections; + const MultipleCollectionAccessor& _collections; PlanYieldPolicySBE* const _yieldPolicy{nullptr}; diff --git a/src/mongo/db/query/sbe_stage_builder_test_fixture.cpp b/src/mongo/db/query/sbe_stage_builder_test_fixture.cpp index 758f94634f4..1623af88d51 100644 --- a/src/mongo/db/query/sbe_stage_builder_test_fixture.cpp +++ b/src/mongo/db/query/sbe_stage_builder_test_fixture.cpp @@ -61,7 +61,7 @@ SbeStageBuilderTestFixture::buildPlanStage( ASSERT_OK(statusWithCQ.getStatus()); stage_builder::SlotBasedStageBuilder builder{opCtx(), - MultiCollection(CollectionPtr::null), + MultipleCollectionAccessor(CollectionPtr::null), *statusWithCQ.getValue(), *querySolution, nullptr /* YieldPolicy */, diff --git a/src/mongo/db/query/sbe_sub_planner.h b/src/mongo/db/query/sbe_sub_planner.h index b81881006d3..78a462dd598 100644 --- a/src/mongo/db/query/sbe_sub_planner.h +++ b/src/mongo/db/query/sbe_sub_planner.h @@ -45,7 +45,7 @@ namespace mongo::sbe { class SubPlanner final : public BaseRuntimePlanner { public: SubPlanner(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QueryPlannerParams& queryParams, PlanYieldPolicySBE* yieldPolicy) diff --git a/src/mongo/db/query/stage_builder_util.cpp b/src/mongo/db/query/stage_builder_util.cpp index ed41d080153..645f2686084 100644 --- a/src/mongo/db/query/stage_builder_util.cpp +++ b/src/mongo/db/query/stage_builder_util.cpp @@ -55,7 +55,7 @@ std::unique_ptr<PlanStage> buildClassicExecutableTree(OperationContext* opCtx, std::pair<std::unique_ptr<sbe::PlanStage>, stage_builder::PlanStageData> buildSlotBasedExecutableTree(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QuerySolution& solution, PlanYieldPolicy* yieldPolicy) { diff --git a/src/mongo/db/query/stage_builder_util.h b/src/mongo/db/query/stage_builder_util.h index c16c04fd99b..943334e590c 100644 --- a/src/mongo/db/query/stage_builder_util.h +++ b/src/mongo/db/query/stage_builder_util.h @@ -52,7 +52,7 @@ std::unique_ptr<PlanStage> buildClassicExecutableTree(OperationContext* opCtx, std::pair<std::unique_ptr<sbe::PlanStage>, stage_builder::PlanStageData> buildSlotBasedExecutableTree(OperationContext* opCtx, - const MultiCollection& collections, + const MultipleCollectionAccessor& collections, const CanonicalQuery& cq, const QuerySolution& solution, PlanYieldPolicy* yieldPolicy); |