diff options
author | Milena Ivanova <milena.ivanova@mongodb.com> | 2021-03-23 11:24:30 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-04-06 17:49:45 +0000 |
commit | 2dbc4a91be463c37a1a38396b6c52a06d67042fb (patch) | |
tree | 028e04a82dc37317f8bb519627b3482501a418a1 | |
parent | 8222bef16ae26297f7313274e4ca15eb1dd54655 (diff) | |
download | mongo-2dbc4a91be463c37a1a38396b6c52a06d67042fb.tar.gz |
SERVER-54001 Allow rewrite to push computed meta projections past $unpackBucket
Split $addFields to two parts, dependent and independent from the meta field,
and push the dependent part past $unpackBucket.
12 files changed, 455 insertions, 278 deletions
diff --git a/src/mongo/db/exec/add_fields_projection_executor.h b/src/mongo/db/exec/add_fields_projection_executor.h index 0245c22682a..39fe73946ab 100644 --- a/src/mongo/db/exec/add_fields_projection_executor.h +++ b/src/mongo/db/exec/add_fields_projection_executor.h @@ -131,10 +131,11 @@ public: return boost::none; } - void substituteFieldPathElement(const StringData& oldName, const StringData& newName) final { - StringMap<std::string> renames; - renames[oldName] = newName.toString(); - _root->substituteFieldPathElement(renames); + std::pair<BSONObj, bool> extractComputedProjections( + const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames) final { + return _root->extractComputedProjectionsInAddFields(oldName, newName, reservedNames); } private: diff --git a/src/mongo/db/exec/add_fields_projection_executor_test.cpp b/src/mongo/db/exec/add_fields_projection_executor_test.cpp index f4da89e0bca..007e5cc1ffb 100644 --- a/src/mongo/db/exec/add_fields_projection_executor_test.cpp +++ b/src/mongo/db/exec/add_fields_projection_executor_test.cpp @@ -604,5 +604,86 @@ TEST(AddFieldsProjectionExecutorExecutionTest, AlwaysKeepsMetadataFromOriginalDo expectedDoc.copyMetaDataFrom(inputDoc); ASSERT_DOCUMENT_EQ(result, expectedDoc.freeze()); } + +// Extract computed projections depending on a given field. +TEST(AddFieldsProjectionExecutorExecutionTest, ExtractComputedProjections) { + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + AddFieldsProjectionExecutor addFields(expCtx); + addFields.parse(BSON("meta1" << BSON("$toUpper" + << "$myMeta.x"))); + + const std::set<StringData> reservedNames{}; + auto [extractedAddFields, deleteFlag] = + addFields.extractComputedProjections("myMeta", "meta", reservedNames); + + ASSERT_EQ(extractedAddFields.nFields(), 1); + ASSERT_EQ(deleteFlag, true); + auto expectedAddFields = BSON("meta1" << BSON("$toUpper" << BSON_ARRAY("$meta.x"))); + ASSERT_BSONOBJ_EQ(expectedAddFields, extractedAddFields); + + ASSERT_DOCUMENT_EQ(Document(fromjson("{}")), addFields.serializeTransformation(boost::none)); +} + +TEST(AddFieldsProjectionExecutorExecutionTest, ExtractComputedProjectionsPrefix) { + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + AddFieldsProjectionExecutor addFields(expCtx); + addFields.parse(BSON("meta1" << BSON("$toUpper" + << "$myMeta.x") + << "computed2" << BSON("$add" << BSON_ARRAY("$c" << 1)))); + + const std::set<StringData> reservedNames{}; + auto [extractedAddFields, deleteFlag] = + addFields.extractComputedProjections("myMeta", "meta", reservedNames); + + ASSERT_EQ(extractedAddFields.nFields(), 1); + ASSERT_EQ(deleteFlag, false); + auto expectedAddFields = BSON("meta1" << BSON("$toUpper" << BSON_ARRAY("$meta.x"))); + ASSERT_BSONOBJ_EQ(expectedAddFields, extractedAddFields); + + ASSERT_DOCUMENT_EQ(Document(fromjson("{computed2: {$add: [\"$c\", {$const: 1}]}}")), + addFields.serializeTransformation(boost::none)); +} + +TEST(AddFieldsProjectionExecutorExecutionTest, DoNotExtractComputedProjectionsSuffix) { + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + AddFieldsProjectionExecutor addFields(expCtx); + addFields.parse(BSON("computed1" << BSON("$add" << BSON_ARRAY("$c" << 1)) << "meta2" + << BSON("$toUpper" + << "$myMeta.x"))); + + const std::set<StringData> reservedNames{}; + auto [extractedAddFields, deleteFlag] = + addFields.extractComputedProjections("myMeta", "meta", reservedNames); + + ASSERT_EQ(extractedAddFields.nFields(), 0); + ASSERT_EQ(deleteFlag, false); + + auto expectedResult = Document(fromjson( + "{computed1: {$add: [\"$c\", {$const: 1}]}, meta2 : {$toUpper : [\"$myMeta.x\"]}}")); + ASSERT_DOCUMENT_EQ(expectedResult, addFields.serializeTransformation(boost::none)); +} + +TEST(AddFieldsProjectionExecutorExecutionTest, DoNotExtractComputedProjectionWithReservedName) { + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + AddFieldsProjectionExecutor addFields(expCtx); + addFields.parse(BSON("meta1" << BSON("$toUpper" + << "$myMeta.x") + << "data" + << BSON("$toUpper" + << "$myMeta.y"))); + + const std::set<StringData> reservedNames{"data"}; + auto [extractedAddFields, deleteFlag] = + addFields.extractComputedProjections("myMeta", "meta", reservedNames); + + ASSERT_EQ(extractedAddFields.nFields(), 1); + ASSERT_EQ(deleteFlag, false); + auto expectedAddFields = BSON("meta1" << BSON("$toUpper" << BSON_ARRAY("$meta.x"))); + ASSERT_BSONOBJ_EQ(expectedAddFields, extractedAddFields); + + ASSERT_DOCUMENT_EQ(Document(fromjson("{data: {$toUpper: [\"$myMeta.y\"]}}")), + addFields.serializeTransformation(boost::none)); +} + } // namespace } // namespace mongo::projection_executor diff --git a/src/mongo/db/exec/inclusion_projection_executor.cpp b/src/mongo/db/exec/inclusion_projection_executor.cpp index 71e80b33709..81ec31c02b2 100644 --- a/src/mongo/db/exec/inclusion_projection_executor.cpp +++ b/src/mongo/db/exec/inclusion_projection_executor.cpp @@ -32,6 +32,8 @@ #include "mongo/db/exec/inclusion_projection_executor.h" namespace mongo::projection_executor { +using ComputedFieldsPolicy = ProjectionPolicies::ComputedFieldsPolicy; + Document FastPathEligibleInclusionNode::applyToDocument(const Document& inputDoc) const { // A fast-path inclusion projection supports inclusion-only fields, so make sure we have no // computed fields in the specification. @@ -87,6 +89,151 @@ void FastPathEligibleInclusionNode::_applyProjections(BSONObj bson, BSONObjBuild } } +namespace { +// A helper function to substitute field path element in expression using the 'renames' map. +boost::intrusive_ptr<Expression> substituteInExpr(boost::intrusive_ptr<Expression> ex, + StringMap<std::string> renames) { + SubstituteFieldPathWalker substituteWalker(renames); + auto substExpr = expression_walker::walk(&substituteWalker, ex.get()); + if (substExpr.get() != nullptr) { + return substExpr.release(); + } + return ex; +}; +} // namespace + +std::pair<BSONObj, bool> InclusionNode::extractComputedProjectionsInProject( + const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames) { + if (_policies.computedFieldsPolicy != ComputedFieldsPolicy::kAllowComputedFields) { + return {BSONObj{}, false}; + } + // Auxiliary vector with extracted computed projections: <name, expression, replacement + // strategy>. If the replacement strategy flag is true, the expression is replaced with a + // projected field. If it is false - the expression is replaced with an identity projection. + std::vector<std::tuple<StringData, boost::intrusive_ptr<Expression>, bool>> + addFieldsExpressions; + bool replaceWithProjField = true; + for (auto&& field : _orderToProcessAdditionsAndChildren) { + if (reservedNames.count(field) > 0) { + // Do not pushdown computed projection with reserved name. + replaceWithProjField = false; + continue; + } + auto expressionIt = _expressions.find(field); + if (expressionIt == _expressions.end()) { + // After seeing the first dotted path expression we need to replace computed + // projections with identity projections to preserve the field order. + replaceWithProjField = false; + continue; + } + DepsTracker deps; + expressionIt->second->addDependencies(&deps); + auto topLevelFieldNames = + deps.toProjectionWithoutMetadata(DepsTracker::TruncateToRootLevel::yes) + .getFieldNames<std::set<std::string>>(); + topLevelFieldNames.erase("_id"); + + if (topLevelFieldNames.size() == 1 && topLevelFieldNames.count(oldName.toString()) == 1) { + // Substitute newName for oldName in the expression. + StringMap<std::string> renames; + renames[oldName] = newName.toString(); + addFieldsExpressions.emplace_back(expressionIt->first, + substituteInExpr(expressionIt->second, renames), + replaceWithProjField); + } else { + // After seeing a computed expression that depends on other fields, we need to preserve + // the order by replacing following computed projections with identity projections. + replaceWithProjField = false; + } + } + + if (!addFieldsExpressions.empty()) { + BSONObjBuilder bb; + for (const auto& expressionSpec : addFieldsExpressions) { + auto&& fieldName = std::get<0>(expressionSpec).toString(); + auto oldExpr = std::get<1>(expressionSpec); + oldExpr->serialize(false).addToBsonObj(&bb, fieldName); + + if (std::get<2>(expressionSpec)) { + // Replace the expression with an inclusion projected field. + _projectedFields.insert(fieldName); + _expressions.erase(fieldName); + // Only computed projections at the beginning of the list were marked to become + // projected fields. The new projected field is at the beginning of the + // _orderToProcessAdditionsAndChildren list. + _orderToProcessAdditionsAndChildren.erase( + _orderToProcessAdditionsAndChildren.begin()); + } else { + // Replace the expression with identity projection. + auto newExpr = ExpressionFieldPath::createPathFromString( + oldExpr->getExpressionContext(), + fieldName, + oldExpr->getExpressionContext()->variablesParseState); + _expressions[fieldName] = newExpr; + } + } + return {bb.obj(), false}; + } + return {BSONObj{}, false}; +} + +std::pair<BSONObj, bool> InclusionNode::extractComputedProjectionsInAddFields( + const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames) { + if (_policies.computedFieldsPolicy != ComputedFieldsPolicy::kAllowComputedFields) { + return {BSONObj{}, false}; + } + // Auxiliary vector with extracted computed projections: <name, expression>. + // To preserve the original fields order, only projections at the beginning of the + // _orderToProcessAdditionsAndChildren list can be extracted for pushdown. + std::vector<std::pair<StringData, boost::intrusive_ptr<Expression>>> addFieldsExpressions; + for (auto&& field : _orderToProcessAdditionsAndChildren) { + // Do not extract for pushdown computed projection with reserved name. + if (reservedNames.count(field) > 0) { + break; + } + auto expressionIt = _expressions.find(field); + if (expressionIt == _expressions.end()) { + break; + } + DepsTracker deps; + expressionIt->second->addDependencies(&deps); + auto topLevelFieldNames = + deps.toProjectionWithoutMetadata(DepsTracker::TruncateToRootLevel::yes) + .getFieldNames<std::set<std::string>>(); + topLevelFieldNames.erase("_id"); + + if (topLevelFieldNames.size() == 1 && topLevelFieldNames.count(oldName.toString()) == 1) { + // Substitute newName for oldName in the expression. + StringMap<std::string> renames; + renames[oldName] = newName.toString(); + addFieldsExpressions.emplace_back(expressionIt->first, + substituteInExpr(expressionIt->second, renames)); + } else { + break; + } + } + + if (!addFieldsExpressions.empty()) { + BSONObjBuilder bb; + for (const auto& expressionSpec : addFieldsExpressions) { + auto&& fieldName = expressionSpec.first.toString(); + auto expr = expressionSpec.second; + expr->serialize(false).addToBsonObj(&bb, fieldName); + + // Remove the expression from this inclusion node. + _expressions.erase(fieldName); + _orderToProcessAdditionsAndChildren.erase(_orderToProcessAdditionsAndChildren.begin()); + } + // If all expressions have been extracted, this inclusion node should be removed. + return {bb.obj(), _orderToProcessAdditionsAndChildren.size() == 0}; + } + return {BSONObj{}, false}; +} + void FastPathEligibleInclusionNode::_applyProjectionsToArray(BSONObj array, BSONArrayBuilder* bab) const { BSONObjIterator it{array}; diff --git a/src/mongo/db/exec/inclusion_projection_executor.h b/src/mongo/db/exec/inclusion_projection_executor.h index 77aa56dbac4..a0429ff924f 100644 --- a/src/mongo/db/exec/inclusion_projection_executor.h +++ b/src/mongo/db/exec/inclusion_projection_executor.h @@ -76,28 +76,38 @@ public: return _children.size() + _projectedFields.size(); } + // The following two methods extract from the InclusionNode computed projections that depend + // only on the 'oldName' field. We need two versions for $project and $addFields, due to + // different functionality: we need to replace the fields in $project to not lose them, and we + // can just remove them from $addFields. /** - * All field paths with the first path element in the 'renames' map are substituted for field - * paths with respective mapped name as a first element. The change is applied to all - * expressions of the InclusionNode, including the expressions in its children. - * + * Returns a pair of <BSONObj, bool>. The BSONObj contains extracted computed projections that + * depend only on the 'oldName' field and is empty if no such projection exists. In the + * extracted expressions the 'oldName' is substituted for the 'newName'. If a projection name + * is in the 'reservedNames' set, it is ineligible for extraction. Each extracted computed + * projection is replaced with a projected field or with an identity projection. + * The returned boolean flag is always false meaning that the original projection is not empty + * and cannot be deleted. */ - void substituteFieldPathElement(const StringMap<std::string>& renames) { - SubstituteFieldPathWalker substituteWalker(renames); - for (auto&& expressionPair : _expressions) { - auto substExpr = - expression_walker::walk(&substituteWalker, expressionPair.second.get()); - if (substExpr.get() != nullptr) { - expressionPair.second = substExpr.release(); - } - } - - for (auto&& childPair : _children) { - static_cast<InclusionNode*>(childPair.second.get()) - ->substituteFieldPathElement(renames); - } - } + std::pair<BSONObj, bool> extractComputedProjectionsInProject( + const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames); + /** + * Returns a pair of <BSONObj, bool>. The BSONObj contains extracted computed projections that + * depend only on the 'oldName' field and is empty if no such projection exists. In the + * extracted expressions the 'oldName' is substituted for the 'newName'. If a projection name + * is in the 'reservedNames' set, it is ineligible for extraction. To preserve the original + * field order the extraction stops when reaching a field which cannot be extracted. The + * extracted projections are removed from the node. + * The returned boolean flag is true if the original projection has become empty after the + * extraction and can be deleted by the caller. + */ + std::pair<BSONObj, bool> extractComputedProjectionsInAddFields( + const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames); protected: // For inclusions, we can apply an optimization here by simply appending to the output document @@ -267,10 +277,11 @@ public: return exhaustivePaths; } - BSONObj extractComputedProjections(const std::string& oldName, - const std::string& newName, - const std::set<StringData>& reservedNames) final { - return _root->extractComputedProjections(oldName, newName, reservedNames); + std::pair<BSONObj, bool> extractComputedProjections( + const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames) final { + return _root->extractComputedProjectionsInProject(oldName, newName, reservedNames); } private: diff --git a/src/mongo/db/exec/inclusion_projection_executor_test.cpp b/src/mongo/db/exec/inclusion_projection_executor_test.cpp index 9e94fac4910..029b3b091c1 100644 --- a/src/mongo/db/exec/inclusion_projection_executor_test.cpp +++ b/src/mongo/db/exec/inclusion_projection_executor_test.cpp @@ -1055,13 +1055,15 @@ TEST_F(InclusionProjectionExecutionTestWithFallBackToDefault, ExtractComputedPro auto r = static_cast<InclusionProjectionExecutor*>(inclusion.get())->getRoot(); const std::set<StringData> reservedNames{}; - auto addFields = r->extractComputedProjections("myMeta", "meta", reservedNames); + auto [addFields, deleteFlag] = + r->extractComputedProjectionsInProject("myMeta", "meta", reservedNames); ASSERT_EQ(addFields.nFields(), 2); auto expectedAddFields = BSON("computedMeta1" << BSON("$toUpper" << BSON_ARRAY("$meta.x")) << "computedMeta3" << "$meta"); ASSERT_BSONOBJ_EQ(expectedAddFields, addFields); + ASSERT_EQ(deleteFlag, false); auto expectedProjection = Document(fromjson("{_id: true, computedMeta1: true, computed2: {$add: [\"$c\", {$const: " @@ -1080,7 +1082,8 @@ TEST_F(InclusionProjectionExecutionTestWithFallBackToDefault, ApplyProjectionAft auto r = static_cast<InclusionProjectionExecutor*>(inclusion.get())->getRoot(); const std::set<StringData> reservedNames{}; - auto addFields = r->extractComputedProjections("myMeta", "meta", reservedNames); + auto [addFields, deleteFlag] = + r->extractComputedProjectionsInProject("myMeta", "meta", reservedNames); // Assuming the document was produced by the $_internalUnpackBucket. auto result = inclusion->applyTransformation( @@ -1100,12 +1103,14 @@ TEST_F(InclusionProjectionExecutionTestWithFallBackToDefault, DoNotExtractReserv auto r = static_cast<InclusionProjectionExecutor*>(inclusion.get())->getRoot(); const std::set<StringData> reservedNames{"meta", "data", "_id"}; - auto addFields = r->extractComputedProjections("myMeta", "meta", reservedNames); + auto [addFields, deleteFlag] = + r->extractComputedProjectionsInProject("myMeta", "meta", reservedNames); ASSERT_EQ(addFields.nFields(), 1); auto expectedAddFields = BSON("newMeta" << "$meta"); ASSERT_BSONOBJ_EQ(expectedAddFields, addFields); + ASSERT_EQ(deleteFlag, false); auto expectedProjection = Document(fromjson( "{_id: true, a: true, data: {\"$toUpper\" : [\"$myMeta.x\"]}, newMeta: \"$newMeta\"}")); diff --git a/src/mongo/db/exec/projection_node.cpp b/src/mongo/db/exec/projection_node.cpp index 29385c0e9d8..0cee88a78f6 100644 --- a/src/mongo/db/exec/projection_node.cpp +++ b/src/mongo/db/exec/projection_node.cpp @@ -30,7 +30,6 @@ #include "mongo/platform/basic.h" #include "mongo/db/exec/projection_node.h" -#include "mongo/db/pipeline/expression_walker.h" namespace mongo::projection_executor { using ArrayRecursionPolicy = ProjectionPolicies::ArrayRecursionPolicy; @@ -306,90 +305,4 @@ void ProjectionNode::serialize(boost::optional<ExplainOptions::Verbosity> explai } } } - -BSONObj ProjectionNode::extractComputedProjections(const StringData& oldName, - const StringData& newName, - const std::set<StringData>& reservedNames) { - if (_policies.computedFieldsPolicy != ComputedFieldsPolicy::kAllowComputedFields) { - return BSONObj{}; - } - // Auxiliary vector with extracted computed projections: <name, expression, replacement - // strategy>. If the replacement strategy flag is true, the expression is replaced with a - // projected field. If it is false - the expression is replaced with an identity projection. - std::vector<std::tuple<StringData, boost::intrusive_ptr<Expression>, bool>> - addFieldsExpressions; - bool replaceWithProjField = true; - for (auto&& field : _orderToProcessAdditionsAndChildren) { - if (reservedNames.count(field) > 0) { - // Do not pushdown computed projection with reserved name. - replaceWithProjField = false; - continue; - } - auto expressionIt = _expressions.find(field); - if (expressionIt == _expressions.end()) { - // After seeing the first dotted path expression we need to replace computed - // projections with identity projections to preserve the field order. - replaceWithProjField = false; - continue; - } - DepsTracker deps; - expressionIt->second->addDependencies(&deps); - auto topLevelFieldNames = - deps.toProjectionWithoutMetadata(DepsTracker::TruncateToRootLevel::yes) - .getFieldNames<std::set<std::string>>(); - topLevelFieldNames.erase("_id"); - - if (topLevelFieldNames.size() == 1 && topLevelFieldNames.count(oldName.toString()) == 1) { - // Substitute newName for oldName in the expression. - StringMap<std::string> renames; - renames[oldName] = newName.toString(); - auto substituteInExpr = - [&renames]( - boost::intrusive_ptr<Expression> ex) -> boost::intrusive_ptr<Expression> { - SubstituteFieldPathWalker substituteWalker(renames); - auto substExpr = expression_walker::walk(&substituteWalker, ex.get()); - if (substExpr.get() != nullptr) { - return substExpr.release(); - } - return ex; - }; - addFieldsExpressions.emplace_back( - expressionIt->first, substituteInExpr(expressionIt->second), replaceWithProjField); - } else { - // After seeing a computed expression that depends on other fields, we need to preserve - // the order by replacing following computed projections with identity projections. - replaceWithProjField = false; - } - } - - if (!addFieldsExpressions.empty()) { - BSONObjBuilder bb; - for (const auto& expressionSpec : addFieldsExpressions) { - auto&& fieldName = std::get<0>(expressionSpec).toString(); - auto oldExpr = std::get<1>(expressionSpec); - oldExpr->serialize(false).addToBsonObj(&bb, fieldName); - - if (std::get<2>(expressionSpec)) { - // Replace the expression with an inclusion projected field. - _projectedFields.insert(fieldName); - _expressions.erase(fieldName); - // Only computed projections at the beginning of the list were marked to become - // projected fields. The new projected field is at the beginning of the - // _orderToProcessAdditionsAndChildren list. - _orderToProcessAdditionsAndChildren.erase( - _orderToProcessAdditionsAndChildren.begin()); - } else { - // Replace the expression with identity projection. - auto newExpr = ExpressionFieldPath::createPathFromString( - oldExpr->getExpressionContext(), - fieldName, - oldExpr->getExpressionContext()->variablesParseState); - _expressions[fieldName] = newExpr; - } - } - return bb.obj(); - } - return BSONObj{}; -} - } // namespace mongo::projection_executor diff --git a/src/mongo/db/exec/projection_node.h b/src/mongo/db/exec/projection_node.h index 0bc61927438..e6a4feab973 100644 --- a/src/mongo/db/exec/projection_node.h +++ b/src/mongo/db/exec/projection_node.h @@ -133,17 +133,6 @@ public: void serialize(boost::optional<ExplainOptions::Verbosity> explain, MutableDocument* output) const; - /** - * Extract computed projections that depend only on the 'oldName' field. Extraction is not - * allowed if the name of the projection is in the 'reservedNames' set. - * The function returns a BSONObj with computed projections, if such exist, or an empty BSONObj. - * Each extracted computed projection is replaced with a projected field or with an identity - * projection. - */ - BSONObj extractComputedProjections(const StringData& oldName, - const StringData& newName, - const std::set<StringData>& reservedNames); - protected: /** * Creates the child if it doesn't already exist. 'field' is not allowed to be dotted. Returns @@ -180,6 +169,16 @@ protected: // Whether this node or any child of this node contains a computed field. bool _subtreeContainsComputedFields{false}; + // Our projection semantics are such that all field additions need to be processed in the order + // specified. '_orderToProcessAdditionsAndChildren' tracks that order. + // + // For example, for the specification {a: <expression>, "b.c": <expression>, d: <expression>}, + // we need to add the top level fields in the order "a", then "b", then "d". This ordering + // information needs to be tracked separately, since "a" and "d" will be tracked via + // '_expressions', and "b.c" will be tracked as a child ProjectionNode in '_children'. For the + // example above, '_orderToProcessAdditionsAndChildren' would be ["a", "b", "d"]. + std::vector<std::string> _orderToProcessAdditionsAndChildren; + private: // Iterates 'inputDoc' for each projected field, adding to or removing from 'outputDoc'. Also // copies over enough information to preserve the structure of the incoming document for the @@ -223,16 +222,6 @@ private: */ void _addProjectionForPath(const FieldPath& path); - // Our projection semantics are such that all field additions need to be processed in the order - // specified. '_orderToProcessAdditionsAndChildren' tracks that order. - // - // For example, for the specification {a: <expression>, "b.c": <expression>, d: <expression>}, - // we need to add the top level fields in the order "a", then "b", then "d". This ordering - // information needs to be tracked separately, since "a" and "d" will be tracked via - // '_expressions', and "b.c" will be tracked as a child ProjectionNode in '_children'. For the - // example above, '_orderToProcessAdditionsAndChildren' would be ["a", "b", "d"]. - std::vector<std::string> _orderToProcessAdditionsAndChildren; - // Maximum number of fields that need to be projected. This allows for an "early" return // optimization which means we don't have to iterate over an entire document. The value is // stored here to avoid re-computation for each document. diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp index 0d6e28736f1..8e2e81d2d14 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp @@ -94,6 +94,24 @@ auto determineIncludeField(StringData fieldName, } /** + * Erase computed meta projection fields if they are present in the exclusion field set. + */ +void eraseExcludedComputedMetaProjFields(BucketUnpacker::Behavior unpackerBehavior, + BucketSpec* bucketSpec) { + if (unpackerBehavior == BucketUnpacker::Behavior::kExclude && + bucketSpec->computedMetaProjFields.size() > 0) { + for (auto it = bucketSpec->computedMetaProjFields.begin(); + it != bucketSpec->computedMetaProjFields.end();) { + if (bucketSpec->fieldSet.find(*it) != bucketSpec->fieldSet.end()) { + it = bucketSpec->computedMetaProjFields.erase(it); + } else { + it++; + } + } + } +} + +/** * A projection can be internalized if every field corresponds to a boolean value. Note that this * correctly rejects dotted fieldnames, which are mapped to objects internally. */ @@ -304,7 +322,6 @@ void BucketUnpacker::reset(BSONObj&& bucket) { } } - // Update computed meta projections with values from this bucket. if (!_spec.computedMetaProjFields.empty()) { for (auto&& name : _spec.computedMetaProjFields) { @@ -320,6 +337,7 @@ void BucketUnpacker::setBucketSpecAndBehavior(BucketSpec&& bucketSpec, Behavior _includeMetaField = eraseMetaFromFieldSetAndDetermineIncludeMeta(behavior, &bucketSpec); _includeTimeField = determineIncludeTimeField(behavior, &bucketSpec); _unpackerBehavior = behavior; + eraseExcludedComputedMetaProjFields(behavior, &bucketSpec); _spec = std::move(bucketSpec); } @@ -587,42 +605,56 @@ DocumentSource::GetNextResult DocumentSourceInternalUnpackBucket::doGetNext() { return nextResult; } -void DocumentSourceInternalUnpackBucket::pushDownComputedMetaProjection( +bool DocumentSourceInternalUnpackBucket::pushDownComputedMetaProjection( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { + bool nextStageWasRemoved = false; if (std::next(itr) == container->end()) { - return; - } - if (!_bucketUnpacker.bucketSpec().metaField || !_bucketUnpacker.includeMetaField()) { - return; + return nextStageWasRemoved; } - auto nextProject = - dynamic_cast<DocumentSourceSingleDocumentTransformation*>((*std::next(itr)).get()); - if (!nextProject || - nextProject->getType() != TransformerInterface::TransformerType::kInclusionProjection) { - return; + if (!_bucketUnpacker.bucketSpec().metaField) { + return nextStageWasRemoved; } - auto& metaName = _bucketUnpacker.bucketSpec().metaField.get(); - auto addFieldsSpec = - nextProject->extractComputedProjections(metaName, - timeseries::kBucketMetaFieldName.toString(), - BucketUnpacker::reservedBucketFieldNames); - if (!addFieldsSpec.isEmpty()) { - std::vector<StringData> computedMetaProjFields; - for (auto&& elem : addFieldsSpec) { - computedMetaProjFields.emplace_back(elem.fieldName()); + if (auto nextTransform = + dynamic_cast<DocumentSourceSingleDocumentTransformation*>(std::next(itr)->get()); + nextTransform && + (nextTransform->getType() == TransformerInterface::TransformerType::kInclusionProjection || + nextTransform->getType() == TransformerInterface::TransformerType::kComputedProjection)) { + + auto& metaName = _bucketUnpacker.bucketSpec().metaField.get(); + auto [addFieldsSpec, deleteStage] = + nextTransform->extractComputedProjections(metaName, + timeseries::kBucketMetaFieldName.toString(), + BucketUnpacker::reservedBucketFieldNames); + nextStageWasRemoved = deleteStage; + + if (!addFieldsSpec.isEmpty()) { + // Extend bucket specification of this stage to include the computed meta projections + // that are passed through. + std::vector<StringData> computedMetaProjFields; + for (auto&& elem : addFieldsSpec) { + computedMetaProjFields.emplace_back(elem.fieldName()); + } + _bucketUnpacker.addComputedMetaProjFields(computedMetaProjFields); + // Insert extracted computed projections before the $_internalUnpackBucket. + container->insert( + itr, + DocumentSourceAddFields::createFromBson( + BSON("$addFields" << addFieldsSpec).firstElement(), getContext())); + // Remove the next stage if it became empty after the field extraction. + if (deleteStage) { + container->erase(std::next(itr)); + } } - _bucketUnpacker.addComputedMetaProjFields(computedMetaProjFields); - container->insert(itr, - DocumentSourceAddFields::createFromBson( - BSON("$addFields" << addFieldsSpec).firstElement(), getContext())); } + return nextStageWasRemoved; } void DocumentSourceInternalUnpackBucket::internalizeProject(const BSONObj& project, bool isInclusion) { // 'fields' are the top-level fields to be included/excluded by the unpacker. We handle the - // special case of _id, which may be excluded in an inclusion $project (or vice versa), here. + // special case of _id, which may be excluded in an inclusion $project (or vice versa), + // here. auto fields = project.getFieldNames<std::set<std::string>>(); if (auto elt = project.getField("_id"); (elt.isBoolean() && elt.Bool() != isInclusion) || (elt.isNumber() && (elt.Int() == 1) != isInclusion)) { @@ -640,7 +672,8 @@ void DocumentSourceInternalUnpackBucket::internalizeProject(const BSONObj& proje std::pair<BSONObj, bool> DocumentSourceInternalUnpackBucket::extractOrBuildProjectToInternalize( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) const { if (std::next(itr) == container->end() || !_bucketUnpacker.bucketSpec().fieldSet.empty()) { - // There is no project to internalize or there are already fields being included/excluded. + // There is no project to internalize or there are already fields being + // included/excluded. return {BSONObj{}, false}; } @@ -651,9 +684,9 @@ std::pair<BSONObj, bool> DocumentSourceInternalUnpackBucket::extractOrBuildProje return {existingProj, isInclusion}; } - // Attempt to get an inclusion $project representing the root-level dependencies of the pipeline - // after the $_internalUnpackBucket. If this $project is not empty, then the dependency set was - // finite. + // Attempt to get an inclusion $project representing the root-level dependencies of the + // pipeline after the $_internalUnpackBucket. If this $project is not empty, then the + // dependency set was finite. Pipeline::SourceContainer restOfPipeline(std::next(itr), container->end()); auto deps = Pipeline::getDependenciesForContainer(pExpCtx, restOfPipeline, boost::none); if (auto dependencyProj = @@ -676,17 +709,18 @@ std::unique_ptr<MatchExpression> createComparisonPredicate( auto path = matchExpr->path(); auto rhs = matchExpr->getData(); - // The control field's min and max are chosen using a field-order insensitive comparator, while - // MatchExpressions use a comparator that treats field-order as significant. Because of this we - // will not perform this optimization on queries with operands of compound types. + // The control field's min and max are chosen using a field-order insensitive comparator, + // while MatchExpressions use a comparator that treats field-order as significant. Because + // of this we will not perform this optimization on queries with operands of compound types. if (rhs.type() == BSONType::Object || rhs.type() == BSONType::Array) { return nullptr; } - // MatchExpressions have special comparison semantics regarding null, in that {$eq: null} will - // match all documents where the field is either null or missing. Because this is different - // from both the comparison semantics that InternalExprComparison expressions and the control's - // min and max fields use, we will not perform this optimization on queries with null operands. + // MatchExpressions have special comparison semantics regarding null, in that {$eq: null} + // will match all documents where the field is either null or missing. Because this is + // different from both the comparison semantics that InternalExprComparison expressions and + // the control's min and max fields use, we will not perform this optimization on queries + // with null operands. if (rhs.type() == BSONType::jstNULL) { return nullptr; } @@ -789,58 +823,6 @@ DocumentSourceInternalUnpackBucket::splitMatchOnMetaAndRename( return {nullptr, match}; } -// Push down computed projections over meta fields ($addFields or $set). -void DocumentSourceInternalUnpackBucket::pushDownAddFieldsMetaProjection( - Pipeline::SourceContainer::iterator itr) { - if (!_bucketUnpacker.bucketSpec().metaField) { - return; - } - auto& metaName = _bucketUnpacker.bucketSpec().metaField.get(); - if (auto nextTransform = - dynamic_cast<DocumentSourceSingleDocumentTransformation*>((*std::next(itr)).get()); - nextTransform && - nextTransform->getType() == TransformerInterface::TransformerType::kComputedProjection) { - // Check if the transformation works exclusively over metafields. - DepsTracker deps; - nextTransform->getDependencies(&deps); - auto topLevelFields = - deps.toProjectionWithoutMetadata(DepsTracker::TruncateToRootLevel::yes); - auto topLevelFieldNames = topLevelFields.getFieldNames<std::set<std::string>>(); - topLevelFieldNames.erase("_id"); - - // Do not pushdown if a computed field name is (or starts with) a reserved bucket field - // name. - auto newNames = nextTransform->getModifiedPaths().getNewNames(); - for (auto&& name : newNames) { - auto pos = name.find('.'); - auto prefix = (pos == std::string::npos) ? name : name.substr(0, pos); - if (BucketUnpacker::isReservedBucketFieldName(prefix)) { - return; - } - } - - if (topLevelFieldNames.size() == 1 && topLevelFieldNames.count(metaName) == 1) { - // a) Rename the user metaField name into bucket's "meta" name in all expressions in - // the transformation phase. - nextTransform->substituteFieldPathElement(metaName, - timeseries::kBucketMetaFieldName.toString()); - - // b) Modify this stage to include the computed meta projections from the next - // stage. - auto computedMetaProj = - nextTransform->getTransformer().serializeTransformation(boost::none).toBson(); - std::vector<StringData> computedMetaProjFields; - for (auto&& elem : computedMetaProj) { - computedMetaProjFields.emplace_back(elem.fieldName()); - } - _bucketUnpacker.addComputedMetaProjFields(computedMetaProjFields); - - // c) Swap the $_internalUnpackBucket stage with the $addFields stage. - std::swap(*itr, *std::next(itr)); - } - } -} - Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::doOptimizeAt( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { invariant(*itr == this); @@ -914,17 +896,12 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::doOptimi } } - // Attempt to extract computed meta projections from subsequent $project and push them before - // the $_internalunpackBucket. + // Attempt to extract computed meta projections from subsequent $project, $addFields, or $set + // and push them before the $_internalunpackBucket. pushDownComputedMetaProjection(itr, container); - // If there is $addFields or $set that operate only on metadata, push it before the - // $_internalUnpackBucket. - if (std::next(itr) != container->end()) { - pushDownAddFieldsMetaProjection(itr); - } - // Attempt to build a $project based on dependency analysis or extract one from the pipeline. We - // can internalize the result so we can handle projections during unpacking. + // Attempt to build a $project based on dependency analysis or extract one from the + // pipeline. We can internalize the result so we can handle projections during unpacking. if (auto [project, isInclusion] = extractOrBuildProjectToInternalize(itr, container); !project.isEmpty()) { internalizeProject(project, isInclusion); diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h index ee8f703a09d..c5133643874 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h @@ -88,9 +88,6 @@ public: // Set of field names reserved for time-series buckets. static const std::set<StringData> reservedBucketFieldNames; - static bool isReservedBucketFieldName(const StringData& name) { - return (reservedBucketFieldNames.count(name) > 0); - } // When BucketUnpacker is created with kInclude it must produce measurements that contain the // set of fields. Otherwise, if the kExclude option is used, the measurements will include the // set difference between all fields in the bucket and the provided fields. @@ -317,9 +314,12 @@ public: return _sampleSize; } - void pushDownAddFieldsMetaProjection(Pipeline::SourceContainer::iterator itr); - - void pushDownComputedMetaProjection(Pipeline::SourceContainer::iterator itr, + /** + * If the stage after $_internalUnpackBucket is $project, $addFields, or $set, try to extract + * from it computed meta projections and push them pass the current stage. Return true if the + * next stage was removed as a result of the optimization. + */ + bool pushDownComputedMetaProjection(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container); private: diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/pushdown_computed_meta_projections_test.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/pushdown_computed_meta_projections_test.cpp index abf095f3552..c2cf0047aac 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/pushdown_computed_meta_projections_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/pushdown_computed_meta_projections_test.cpp @@ -52,8 +52,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownAddFieldsMetaProjection(container.begin()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, true); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(fromjson("{$addFields: {newMeta: {$toUpper: ['$meta']}}}"), serialized[0]); @@ -69,8 +70,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, OptimizeAddFieldsWithMetaPro auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownAddFieldsMetaProjection(container.begin()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, true); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(fromjson("{$addFields: {newMeta: {$concat: ['$meta.a', '$meta.b']}}}"), @@ -87,8 +89,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, OptimizeAddFieldsWith2MetaPr auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownAddFieldsMetaProjection(container.begin()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, true); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(fromjson("{$addFields: {device: '$meta.a', deviceType: '$meta.b'}}"), @@ -96,6 +99,43 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, OptimizeAddFieldsWith2MetaPr ASSERT_BSONOBJ_EQ(unpackSpecObj, serialized[1]); } +TEST_F(InternalUnpackBucketPushdownProjectionsTest, SplitAddFieldsWithMixedProjectionFields) { + auto unpackSpecObj = + fromjson("{$_internalUnpackBucket: { exclude: [], timeField: 'foo', metaField: 'myMeta'}}"); + auto addFieldsSpecObj = + fromjson("{$addFields: {device: '$myMeta.a', temp: {$add: ['$temperature', '$offset']}}}"); + + auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); + auto& container = pipeline->getSources(); + auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + + ASSERT_EQ(nextStageIsRemoved, false); + auto serialized = pipeline->serializeToBson(); + ASSERT_EQ(3u, serialized.size()); + ASSERT_BSONOBJ_EQ(fromjson("{$addFields: {device: '$meta.a'}}"), serialized[0]); + ASSERT_BSONOBJ_EQ(unpackSpecObj, serialized[1]); + ASSERT_BSONOBJ_EQ(fromjson("{$addFields: {temp: {$add: ['$temperature', '$offset']}}}"), + serialized[2]); +} + +TEST_F(InternalUnpackBucketPushdownProjectionsTest, DoNotSplitAddFieldsWithMetaProjectionInSuffix) { + auto unpackSpecObj = + fromjson("{$_internalUnpackBucket: { exclude: [], timeField: 'foo', metaField: 'myMeta'}}"); + auto addFieldsSpecObj = fromjson("{$addFields: {temp: '$temperature', device: '$myMeta.a'}}"); + + auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); + auto& container = pipeline->getSources(); + auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + + ASSERT_EQ(nextStageIsRemoved, false); + auto serialized = pipeline->serializeToBson(); + ASSERT_EQ(2u, serialized.size()); + ASSERT_BSONOBJ_EQ(unpackSpecObj, serialized[0]); + ASSERT_BSONOBJ_EQ(addFieldsSpecObj, serialized[1]); +} + TEST_F(InternalUnpackBucketPushdownProjectionsTest, DoNotOptimizeAddFieldsWithMixedProjection) { auto unpackSpecObj = fromjson("{$_internalUnpackBucket: { exclude: [], timeField: 'foo', metaField: 'myMeta'}}"); @@ -105,8 +145,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, DoNotOptimizeAddFieldsWithMi auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownAddFieldsMetaProjection(container.begin()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(unpackSpecObj, serialized[0]); @@ -120,8 +161,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, DoNotOptimizeAddFieldsWithMi auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownAddFieldsMetaProjection(container.begin()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(unpackSpecObj, serialized[0]); @@ -137,8 +179,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownAddFieldsMetaProjection(container.begin()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(unpackSpecObj, serialized[0]); @@ -154,8 +197,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, addFieldsSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownAddFieldsMetaProjection(container.begin()); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(unpackSpecObj, serialized[0]); @@ -174,8 +218,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownComputedMetaProjection(container.begin(), &container); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(3u, serialized.size()); ASSERT_BSONOBJ_EQ(fromjson("{$addFields: { device: '$meta.a'}}"), serialized[0]); @@ -193,8 +238,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownComputedMetaProjection(container.begin(), &container); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(3u, serialized.size()); ASSERT_BSONOBJ_EQ(fromjson("{$addFields: { device: '$meta.a'}}"), serialized[0]); @@ -213,8 +259,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, DoNotPushDownMixedProjection auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownComputedMetaProjection(container.begin(), &container); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(projectSpecObj, serialized[1]); @@ -230,8 +277,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, projectSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownComputedMetaProjection(container.begin(), &container); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(projectSpecObj, serialized[1]); @@ -246,8 +294,9 @@ TEST_F(InternalUnpackBucketPushdownProjectionsTest, DoNotPushDownNestedProjectio auto pipeline = Pipeline::parse(makeVector(unpackSpecObj, projectSpecObj), getExpCtx()); auto& container = pipeline->getSources(); auto unpack = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.begin()->get()); - unpack->pushDownComputedMetaProjection(container.begin(), &container); + auto nextStageIsRemoved = unpack->pushDownComputedMetaProjection(container.begin(), &container); + ASSERT_EQ(nextStageIsRemoved, false); auto serialized = pipeline->serializeToBson(); ASSERT_EQ(2u, serialized.size()); ASSERT_BSONOBJ_EQ(projectSpecObj, serialized[1]); diff --git a/src/mongo/db/pipeline/document_source_single_document_transformation.h b/src/mongo/db/pipeline/document_source_single_document_transformation.h index d3d57c60e7f..158f9936fdc 100644 --- a/src/mongo/db/pipeline/document_source_single_document_transformation.h +++ b/src/mongo/db/pipeline/document_source_single_document_transformation.h @@ -95,26 +95,25 @@ public: } /** - * Substitute the occurence of 'oldName' as first field path element for a 'newName' in all - * expressions in the transformation object. + * Extract computed projection(s) depending on the 'oldName' argument if the transformation is + * of type inclusion projection or computed projection. Extraction is not allowed if the name of + * the projection is in the 'reservedNames' set. The function returns a pair of <BSONObj, bool>. + * The BSONObj contains the computed projections in which the 'oldName' is substituted for the + * 'newName'. The boolean flag is true if the original projection has become empty after the + * extraction and can be deleted by the caller. + * + * For transformation of type inclusion projection the computed projections are replaced with a + * projected field or an identity projection depending on their position in the order of + * additional fields. + * For transformation of type computed projection the extracted computed projections are + * removed. + * + * The function has no effect for exclusion projections, or if there are no computed + * projections, or the computed expression depends on other fields than the oldName. */ - void substituteFieldPathElement(const std::string& oldName, const std::string& newName) { - _parsedTransform->substituteFieldPathElement(oldName, newName); - } - - /** - * If this transformation is an inclusion projection, the function extracts the computed - * projection(s) depending on the oldName argument. Extraction is not allowed if the name of the - * projection is in the 'reservedNames' set. The computed projection is returned as a BSONObj, - * where the oldName is substituted for the newName. In the original inclusion projection it is - * replaced with a projected field or an identity projection depending on its position in the - * order of additional fields. The function has no effect for exclusion projections, or if there - * are no computed projections, or the computed expression depends on other fields than the - * oldName. - */ - BSONObj extractComputedProjections(const std::string& oldName, - const std::string& newName, - const std::set<StringData>& reservedNames) { + std::pair<BSONObj, bool> extractComputedProjections(const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames) { return _parsedTransform->extractComputedProjections(oldName, newName, reservedNames); } diff --git a/src/mongo/db/pipeline/transformer_interface.h b/src/mongo/db/pipeline/transformer_interface.h index a53f9e7328e..40ae3b0564e 100644 --- a/src/mongo/db/pipeline/transformer_interface.h +++ b/src/mongo/db/pipeline/transformer_interface.h @@ -68,12 +68,17 @@ public: virtual Document serializeTransformation( boost::optional<ExplainOptions::Verbosity> explain) const = 0; - virtual void substituteFieldPathElement(const StringData& oldName, const StringData& newName) {} - - virtual BSONObj extractComputedProjections(const std::string& oldName, - const std::string& newName, - const std::set<StringData>& reservedNames) { - return BSONObj{}; + /** + * Method used by inclusion and add fields projecton executors to extract computed projections + * that depend only on the 'oldName' field. Returns a pair of <BSONObj, bool>. The BSONObj + * contains the extracted projections. The boolean flag is true if the original projection has + * become empty after the extraction and can be deleted by the caller. + */ + virtual std::pair<BSONObj, bool> extractComputedProjections( + const StringData& oldName, + const StringData& newName, + const std::set<StringData>& reservedNames) { + return {BSONObj{}, false}; } }; } // namespace mongo |