diff options
author | Ruoxin Xu <ruoxin.xu@mongodb.com> | 2021-05-12 16:21:22 +0100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-05-14 19:33:20 +0000 |
commit | bffe722953e0adaa7a88b7bfdae29d8ca599fc3d (patch) | |
tree | a9326ced5b848161567a05da88c1f2b6ad0afca7 /src/mongo/db/update | |
parent | 1ab98f5e384deca4eddacbfed26cc05e2c5e72d1 (diff) | |
download | mongo-bffe722953e0adaa7a88b7bfdae29d8ca599fc3d.tar.gz |
SERVER-56755 Collect and expose statistics for insert and updates with dots and dollars fields
Diffstat (limited to 'src/mongo/db/update')
-rw-r--r-- | src/mongo/db/update/array_culling_node.cpp | 15 | ||||
-rw-r--r-- | src/mongo/db/update/array_culling_node.h | 4 | ||||
-rw-r--r-- | src/mongo/db/update/modifier_node.cpp | 47 | ||||
-rw-r--r-- | src/mongo/db/update/modifier_node.h | 7 | ||||
-rw-r--r-- | src/mongo/db/update/object_replace_executor.cpp | 14 | ||||
-rw-r--r-- | src/mongo/db/update/pop_node.cpp | 15 | ||||
-rw-r--r-- | src/mongo/db/update/pop_node.h | 4 | ||||
-rw-r--r-- | src/mongo/db/update/storage_validation.cpp | 72 | ||||
-rw-r--r-- | src/mongo/db/update/storage_validation.h | 21 | ||||
-rw-r--r-- | src/mongo/db/update/unset_node.cpp | 18 | ||||
-rw-r--r-- | src/mongo/db/update/unset_node.h | 4 | ||||
-rw-r--r-- | src/mongo/db/update/update_driver.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/update/update_driver.h | 11 | ||||
-rw-r--r-- | src/mongo/db/update/update_executor.h | 2 | ||||
-rw-r--r-- | src/mongo/db/update/update_object_node.cpp | 2 |
15 files changed, 181 insertions, 58 deletions
diff --git a/src/mongo/db/update/array_culling_node.cpp b/src/mongo/db/update/array_culling_node.cpp index 00a442a8d5f..ef6c85e45d3 100644 --- a/src/mongo/db/update/array_culling_node.cpp +++ b/src/mongo/db/update/array_culling_node.cpp @@ -30,6 +30,7 @@ #include "mongo/platform/basic.h" #include "mongo/db/update/array_culling_node.h" +#include "mongo/db/update/storage_validation.h" namespace mongo { @@ -60,11 +61,21 @@ void ArrayCullingNode::validateUpdate(mutablebson::ConstElement updatedElement, mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const { + ModifyResult modifyResult, + const bool validateForStorage, + bool* containsDotsAndDollarsField) const { invariant(modifyResult == ModifyResult::kNormalUpdate); // Removing elements from an array cannot increase BSON depth or modify a DBRef, so we can - // override validateUpdate to do nothing. + // override validateUpdate to not validate storage constraints but we still want to know if + // there is any field name containing '.'/'$'. + bool doRecursiveCheck = true; + storage_validation::storageValid(updatedElement, + doRecursiveCheck, + recursionLevel, + false, /* allowTopLevelDollarPrefixedFields */ + false, /* should validate for storage */ + containsDotsAndDollarsField); } } // namespace mongo diff --git a/src/mongo/db/update/array_culling_node.h b/src/mongo/db/update/array_culling_node.h index b6f2c1f2edb..c3ab2849689 100644 --- a/src/mongo/db/update/array_culling_node.h +++ b/src/mongo/db/update/array_culling_node.h @@ -52,7 +52,9 @@ public: mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const final; + ModifyResult modifyResult, + const bool validateForStorage, + bool* containsDotsAndDollarsField) const final; void setCollator(const CollatorInterface* collator) final { _matcher->setCollator(collator); diff --git a/src/mongo/db/update/modifier_node.cpp b/src/mongo/db/update/modifier_node.cpp index 28af3da1f4a..c8c3327cc82 100644 --- a/src/mongo/db/update/modifier_node.cpp +++ b/src/mongo/db/update/modifier_node.cpp @@ -197,11 +197,14 @@ UpdateExecutor::ApplyResult ModifierNode::applyToExistingElement( applyResult.indexesAffected = false; } - if (applyParams.validateForStorage) { - const uint32_t recursionLevel = updateNodeApplyParams.pathTaken->size(); - validateUpdate( - applyParams.element, leftSibling, rightSibling, recursionLevel, updateResult); - } + const uint32_t recursionLevel = updateNodeApplyParams.pathTaken->size(); + validateUpdate(applyParams.element, + leftSibling, + rightSibling, + recursionLevel, + updateResult, + applyParams.validateForStorage, + &applyResult.containsDotsAndDollarsField); if (auto logBuilder = updateNodeApplyParams.logBuilder) { logUpdate(logBuilder, @@ -248,15 +251,17 @@ UpdateExecutor::ApplyResult ModifierNode::applyToNonexistentElement( MONGO_UNREACHABLE; // The previous uassertStatusOK should always throw. } - if (applyParams.validateForStorage) { - const uint32_t recursionLevel = updateNodeApplyParams.pathTaken->size() + 1; - mutablebson::ConstElement elementForValidation = statusWithFirstCreatedElem.getValue(); - validateUpdate(elementForValidation, - elementForValidation.leftSibling(), - elementForValidation.rightSibling(), - recursionLevel, - ModifyResult::kCreated); - } + ApplyResult applyResult; + + const uint32_t recursionLevel = updateNodeApplyParams.pathTaken->size() + 1; + mutablebson::ConstElement elementForValidation = statusWithFirstCreatedElem.getValue(); + validateUpdate(elementForValidation, + elementForValidation.leftSibling(), + elementForValidation.rightSibling(), + recursionLevel, + ModifyResult::kCreated, + applyParams.validateForStorage, + &applyResult.containsDotsAndDollarsField); for (auto immutablePath = applyParams.immutablePaths.begin(); immutablePath != applyParams.immutablePaths.end(); @@ -304,8 +309,6 @@ UpdateExecutor::ApplyResult ModifierNode::applyToNonexistentElement( } invariant(fullPathTypes.size() == fullPathFr.numParts()); - ApplyResult applyResult; - // Determine if indexes are affected. If we did not create a new element in an array, check // whether the full path affects indexes. If we did create a new element in an array, check // whether the array itself might affect any indexes. This is necessary because if there is @@ -367,9 +370,17 @@ void ModifierNode::validateUpdate(mutablebson::ConstElement updatedElement, mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const { + ModifyResult modifyResult, + const bool validateForStorage, + bool* containsDotsAndDollarsField) const { const bool doRecursiveCheck = true; - storage_validation::storageValid(updatedElement, doRecursiveCheck, recursionLevel); + + storage_validation::storageValid(updatedElement, + doRecursiveCheck, + recursionLevel, + false, /* allowTopLevelDollarPrefixedFields */ + validateForStorage, + containsDotsAndDollarsField); } void ModifierNode::logUpdate(LogBuilderInterface* logBuilder, diff --git a/src/mongo/db/update/modifier_node.h b/src/mongo/db/update/modifier_node.h index d9063768151..9ae0786c073 100644 --- a/src/mongo/db/update/modifier_node.h +++ b/src/mongo/db/update/modifier_node.h @@ -116,12 +116,17 @@ protected: * - 'recursionLevel' is the document nesting depth of the 'updatedElement' field. * - 'modifyResult' is either the value returned by updateExistingElement() or the value * ModifyResult::kCreated. + * - If 'validateForStorage' is true, we should verify that the updated element is valid for + * storage. + * - 'containsDotsAndDollarsField' is true if 'updatedElement' contains any dots/dollars field. */ virtual void validateUpdate(mutablebson::ConstElement updatedElement, mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const; + ModifyResult modifyResult, + const bool validateForStorage, + bool* containsDotsAndDollarsField) const; /** * ModifierNode::apply() calls this method after validation to create an oplog entry for the diff --git a/src/mongo/db/update/object_replace_executor.cpp b/src/mongo/db/update/object_replace_executor.cpp index 92fc0023dcd..22af1870f62 100644 --- a/src/mongo/db/update/object_replace_executor.cpp +++ b/src/mongo/db/update/object_replace_executor.cpp @@ -103,11 +103,13 @@ UpdateExecutor::ApplyResult ObjectReplaceExecutor::applyReplacementUpdate( invariant(applyParams.element.appendElement(elem)); } - // Validate for storage. - if (applyParams.validateForStorage) { - storage_validation::storageValid(applyParams.element.getDocument(), - allowTopLevelDollarPrefixedFields); - } + ApplyResult ret; + + // Validate for storage and check if the document contains any '.'/'$' field name. + storage_validation::storageValid(applyParams.element.getDocument(), + allowTopLevelDollarPrefixedFields, + applyParams.validateForStorage, + &ret.containsDotsAndDollarsField); // Check immutable paths. for (auto path = applyParams.immutablePaths.begin(); path != applyParams.immutablePaths.end(); @@ -146,7 +148,7 @@ UpdateExecutor::ApplyResult ObjectReplaceExecutor::applyReplacementUpdate( } } - return ApplyResult{}; + return ret; } UpdateExecutor::ApplyResult ObjectReplaceExecutor::applyUpdate(ApplyParams applyParams) const { diff --git a/src/mongo/db/update/pop_node.cpp b/src/mongo/db/update/pop_node.cpp index dd18e8e768b..c15f26fe790 100644 --- a/src/mongo/db/update/pop_node.cpp +++ b/src/mongo/db/update/pop_node.cpp @@ -32,6 +32,7 @@ #include "mongo/db/update/pop_node.h" #include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/update/storage_validation.h" namespace mongo { @@ -72,11 +73,21 @@ void PopNode::validateUpdate(mutablebson::ConstElement updatedElement, mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const { + ModifyResult modifyResult, + const bool validateForStorage, + bool* containsDotsAndDollarsField) const { invariant(modifyResult == ModifyResult::kNormalUpdate); // Removing elements from an array cannot increase BSON depth or modify a DBRef, so we can - // override validateUpdate to do nothing. + // override validateUpdate to not validate storage constraints but we still want to know if + // there is any field name containing '.'/'$'. + bool doRecursiveCheck = true; + storage_validation::storageValid(updatedElement, + doRecursiveCheck, + recursionLevel, + false, /* allowTopLevelDollarPrefixedFields */ + false, /* Should validate for storage */ + containsDotsAndDollarsField); } } // namespace mongo diff --git a/src/mongo/db/update/pop_node.h b/src/mongo/db/update/pop_node.h index 3f0f8a55f15..c245d296362 100644 --- a/src/mongo/db/update/pop_node.h +++ b/src/mongo/db/update/pop_node.h @@ -47,7 +47,9 @@ public: mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const final; + ModifyResult modifyResult, + const bool validateForStorage, + bool* containsDotsAndDollarsField) const final; std::unique_ptr<UpdateNode> clone() const final { return std::make_unique<PopNode>(*this); diff --git a/src/mongo/db/update/storage_validation.cpp b/src/mongo/db/update/storage_validation.cpp index ffb70ff21ee..cac577f414c 100644 --- a/src/mongo/db/update/storage_validation.cpp +++ b/src/mongo/db/update/storage_validation.cpp @@ -47,14 +47,21 @@ const StringData idFieldName = "_id"_sd; void storageValidChildren(mutablebson::ConstElement elem, const bool deep, std::uint32_t recursionLevel, - const bool allowTopLevelDollarPrefixes) { + const bool allowTopLevelDollarPrefixes, + const bool shouldValidate, + bool* containsDotsAndDollarsField) { if (!elem.hasChildren()) { return; } auto curr = elem.leftChild(); while (curr.ok()) { - storageValid(curr, deep, recursionLevel + 1, allowTopLevelDollarPrefixes); + storageValid(curr, + deep, + recursionLevel + 1, + allowTopLevelDollarPrefixes, + shouldValidate, + containsDotsAndDollarsField); curr = curr.rightSibling(); } } @@ -118,10 +125,13 @@ void validateDollarPrefixElement(mutablebson::ConstElement elem) { } } // namespace -void storageValid(const mutablebson::Document& doc, const bool allowTopLevelDollarPrefixes) { +void storageValid(const mutablebson::Document& doc, + const bool allowTopLevelDollarPrefixes, + const bool shouldValidate, + bool* containsDotsAndDollarsField) { auto currElem = doc.root().leftChild(); while (currElem.ok()) { - if (currElem.getFieldName() == idFieldName) { + if (currElem.getFieldName() == idFieldName && shouldValidate) { switch (currElem.getType()) { case BSONType::RegEx: case BSONType::Array: @@ -137,7 +147,12 @@ void storageValid(const mutablebson::Document& doc, const bool allowTopLevelDoll // Validate this child element. const auto deep = true; const uint32_t recursionLevel = 1; - storageValid(currElem, deep, recursionLevel, allowTopLevelDollarPrefixes); + storageValid(currElem, + deep, + recursionLevel, + allowTopLevelDollarPrefixes, + shouldValidate, + containsDotsAndDollarsField); currElem = currElem.rightSibling(); } @@ -146,13 +161,17 @@ void storageValid(const mutablebson::Document& doc, const bool allowTopLevelDoll void storageValid(mutablebson::ConstElement elem, const bool deep, std::uint32_t recursionLevel, - const bool allowTopLevelDollarPrefixes) { - uassert(ErrorCodes::BadValue, "Invalid elements cannot be stored.", elem.ok()); - - uassert(ErrorCodes::Overflow, - str::stream() << "Document exceeds maximum nesting depth of " - << BSONDepth::getMaxDepthForUserStorage(), - recursionLevel <= BSONDepth::getMaxDepthForUserStorage()); + const bool allowTopLevelDollarPrefixes, + const bool shouldValidate, + bool* containsDotsAndDollarsField) { + if (shouldValidate) { + uassert(ErrorCodes::BadValue, "Invalid elements cannot be stored.", elem.ok()); + + uassert(ErrorCodes::Overflow, + str::stream() << "Document exceeds maximum nesting depth of " + << BSONDepth::getMaxDepthForUserStorage(), + recursionLevel <= BSONDepth::getMaxDepthForUserStorage()); + } // Field names of elements inside arrays are not meaningful in mutable bson, // so we do not want to validate them. @@ -163,14 +182,20 @@ void storageValid(mutablebson::ConstElement elem, // check top-level fields if 'allowTopLevelDollarPrefixes' is false, and don't validate any // fields for '$'-prefixes if 'allowTopLevelDollarPrefixes' is true. const bool checkTopLevelFields = !allowTopLevelDollarPrefixes && (recursionLevel == 1); - const bool checkFields = - !feature_flags::gFeatureFlagDotsAndDollars.isEnabledAndIgnoreFCV() || checkTopLevelFields; - - if (!childOfArray && checkFields) { - auto fieldName = elem.getFieldName(); - - // Cannot start with "$", unless dbref. - if (fieldName[0] == '$') { + const bool dotsAndDollarsFeatureEnabled = + feature_flags::gFeatureFlagDotsAndDollars.isEnabledAndIgnoreFCV(); + const bool checkFields = !dotsAndDollarsFeatureEnabled || checkTopLevelFields; + + auto fieldName = elem.getFieldName(); + if (fieldName[0] == '$') { + if (dotsAndDollarsFeatureEnabled && containsDotsAndDollarsField) { + *containsDotsAndDollarsField = true; + // If we are not validating for storage, return once a $-prefixed field is found. + if (!shouldValidate) + return; + } + if (!childOfArray && checkFields && shouldValidate) { + // Cannot start with "$", unless dbref. validateDollarPrefixElement(elem); } } @@ -178,7 +203,12 @@ void storageValid(mutablebson::ConstElement elem, if (deep) { // Check children if there are any. - storageValidChildren(elem, deep, recursionLevel, allowTopLevelDollarPrefixes); + storageValidChildren(elem, + deep, + recursionLevel, + allowTopLevelDollarPrefixes, + shouldValidate, + containsDotsAndDollarsField); } } diff --git a/src/mongo/db/update/storage_validation.h b/src/mongo/db/update/storage_validation.h index a16cb8a81d0..9e3ae50ee84 100644 --- a/src/mongo/db/update/storage_validation.h +++ b/src/mongo/db/update/storage_validation.h @@ -43,8 +43,17 @@ namespace storage_validation { * When the dots and dollars feature flag is off, always reject $-prefixed fields. Otherwise, reject * only $-prefixed fields at the top-level of a document. If 'allowTopLevelDollarPrefixes' is set to * true, do not reject $-prefixed fields at the top-level of a document. + * + * 'shouldValidate' is true if the caller wants to validate for storage, otherwise this helper will + * only check top-level $-prefixed field names skipping all the validations. + * + * 'containsDotsAndDollarsField' is set to true if there exists any field name containing '.'/'$' + * during validation. */ -void storageValid(const mutablebson::Document& doc, const bool allowTopLevelDollarPrefixes = false); +void storageValid(const mutablebson::Document& doc, + const bool allowTopLevelDollarPrefixes, + const bool shouldValidate, + bool* containsDotsAndDollarsField); /** * Validates that the MutableBSON element 'elem' is acceptable for storage in a collection. If @@ -54,11 +63,19 @@ void storageValid(const mutablebson::Document& doc, const bool allowTopLevelDoll * When the dots and dollars feature flag is off, always reject $-prefixed fields. Otherwise, reject * only $-prefixed fields at the top-level of a document. If 'allowTopLevelDollarPrefixes' is set to * true, do not reject $-prefixed fields at the top-level of a document. + * + * 'shouldValidate' is true if the caller wants to validate for storage, otherwise this helper will + * only check top-level $-prefixed field names skipping all the validations. + * + * 'containsDotsAndDollarsField' is set to true if there exists any field name containing '.'/'$' + * during validation. */ void storageValid(mutablebson::ConstElement elem, const bool deep, std::uint32_t recursionLevel, - const bool allowTopLevelDollarPrefixes = false); + const bool allowTopLevelDollarPrefixes, + const bool shouldValidate, + bool* containsDotsAndDollarsField); } // namespace storage_validation diff --git a/src/mongo/db/update/unset_node.cpp b/src/mongo/db/update/unset_node.cpp index c731833cfce..75612dba4be 100644 --- a/src/mongo/db/update/unset_node.cpp +++ b/src/mongo/db/update/unset_node.cpp @@ -61,7 +61,9 @@ void UnsetNode::validateUpdate(mutablebson::ConstElement updatedElement, mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const { + ModifyResult modifyResult, + bool validateForStorage, + bool* containsDotsAndDollarsField) const { invariant(modifyResult == ModifyResult::kNormalUpdate); // We only need to check the left and right sibling to see if the removed element was part of a @@ -70,11 +72,21 @@ void UnsetNode::validateUpdate(mutablebson::ConstElement updatedElement, const uint32_t recursionLevelForCheck = 0; if (leftSibling.ok()) { - storage_validation::storageValid(leftSibling, doRecursiveCheck, recursionLevelForCheck); + storage_validation::storageValid(leftSibling, + doRecursiveCheck, + recursionLevelForCheck, + false, /* allowTopLevelDollarPrefixedFields */ + validateForStorage, + containsDotsAndDollarsField); } if (rightSibling.ok()) { - storage_validation::storageValid(rightSibling, doRecursiveCheck, recursionLevelForCheck); + storage_validation::storageValid(rightSibling, + doRecursiveCheck, + recursionLevelForCheck, + false, /* allowTopLevelDollarPrefixedFields */ + validateForStorage, + containsDotsAndDollarsField); } } diff --git a/src/mongo/db/update/unset_node.h b/src/mongo/db/update/unset_node.h index 1aab11bc1b8..bd73f88d790 100644 --- a/src/mongo/db/update/unset_node.h +++ b/src/mongo/db/update/unset_node.h @@ -56,7 +56,9 @@ public: mutablebson::ConstElement leftSibling, mutablebson::ConstElement rightSibling, std::uint32_t recursionLevel, - ModifyResult modifyResult) const final; + ModifyResult modifyResult, + bool validateForStorage, + bool* containsDotsAndDollarsField) const final; void logUpdate(LogBuilderInterface* logBuilder, const RuntimeUpdatePath& pathTaken, diff --git a/src/mongo/db/update/update_driver.cpp b/src/mongo/db/update/update_driver.cpp index 50340b57864..0af1f5739e0 100644 --- a/src/mongo/db/update/update_driver.cpp +++ b/src/mongo/db/update/update_driver.cpp @@ -301,6 +301,9 @@ Status UpdateDriver::update(OperationContext* opCtx, *logOpRec = applyResult.oplogEntry; } + _containsDotsAndDollarsField = + (_containsDotsAndDollarsField || applyResult.containsDotsAndDollarsField); + return Status::OK(); } diff --git a/src/mongo/db/update/update_driver.h b/src/mongo/db/update/update_driver.h index 02765ea844c..e4104a313ec 100644 --- a/src/mongo/db/update/update_driver.h +++ b/src/mongo/db/update/update_driver.h @@ -186,6 +186,14 @@ public: return _positional; } + bool containsDotsAndDollarsField() const { + return _containsDotsAndDollarsField; + } + + void setContainsDotsAndDollarsField(const bool containsDotsAndDollarsField) { + _containsDotsAndDollarsField = containsDotsAndDollarsField; + } + /** * Serialize the update expression to Value. Output of this method is expected to, when parsed, * produce a logically equivalent update expression. @@ -238,6 +246,9 @@ private: // Do any of the mods require positional match details when calling 'prepare'? bool _positional = false; + // True if the updated document contains any '.'/'$' field name. + bool _containsDotsAndDollarsField = false; + // The document used to represent or store the object being updated. mutablebson::Document _objDoc; diff --git a/src/mongo/db/update/update_executor.h b/src/mongo/db/update/update_executor.h index 43d53575d7b..a7ef7dd0837 100644 --- a/src/mongo/db/update/update_executor.h +++ b/src/mongo/db/update/update_executor.h @@ -110,11 +110,13 @@ public: ApplyResult applyResult; applyResult.indexesAffected = false; applyResult.noop = true; + applyResult.containsDotsAndDollarsField = false; return applyResult; } bool indexesAffected = true; bool noop = false; + bool containsDotsAndDollarsField = false; // The oplog entry to log. This is only populated if the operation is not considered a // noop and if the 'logMode' provided in ApplyParams indicates that an oplog entry should diff --git a/src/mongo/db/update/update_object_node.cpp b/src/mongo/db/update/update_object_node.cpp index b27a9fb18e9..256c79d2bae 100644 --- a/src/mongo/db/update/update_object_node.cpp +++ b/src/mongo/db/update/update_object_node.cpp @@ -147,6 +147,8 @@ void applyChild(const UpdateNode& child, applyResult->indexesAffected = applyResult->indexesAffected || childApplyResult.indexesAffected; applyResult->noop = applyResult->noop && childApplyResult.noop; + applyResult->containsDotsAndDollarsField = + applyResult->containsDotsAndDollarsField || childApplyResult.containsDotsAndDollarsField; // Pop 'field' off of 'pathToCreate' or 'pathTaken'. if (!updateNodeApplyParams->pathToCreate->empty()) { |