diff options
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/crypto/fle_crypto.cpp | 9 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto.h | 4 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto_test.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_impl.cpp | 38 | ||||
-rw-r--r-- | src/mongo/db/catalog/document_validation.h | 34 | ||||
-rw-r--r-- | src/mongo/db/commands/find_and_modify.cpp | 13 | ||||
-rw-r--r-- | src/mongo/db/commands/fle_compact_test.cpp | 9 | ||||
-rw-r--r-- | src/mongo/db/commands/write_commands.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/fle_crud.cpp | 94 | ||||
-rw-r--r-- | src/mongo/db/fle_crud.h | 28 | ||||
-rw-r--r-- | src/mongo/db/fle_crud_test.cpp | 47 | ||||
-rw-r--r-- | src/mongo/db/fle_query_interface_mock.cpp | 10 | ||||
-rw-r--r-- | src/mongo/db/fle_query_interface_mock.h | 17 | ||||
-rw-r--r-- | src/mongo/db/ops/write_ops_exec.cpp | 58 | ||||
-rw-r--r-- | src/mongo/db/ops/write_ops_exec.h | 3 | ||||
-rw-r--r-- | src/mongo/idl/generic_argument.idl | 1 | ||||
-rw-r--r-- | src/mongo/s/write_ops/batch_write_op.cpp | 11 |
17 files changed, 306 insertions, 78 deletions
diff --git a/src/mongo/crypto/fle_crypto.cpp b/src/mongo/crypto/fle_crypto.cpp index b7f5eebd3e9..f55db25f970 100644 --- a/src/mongo/crypto/fle_crypto.cpp +++ b/src/mongo/crypto/fle_crypto.cpp @@ -972,7 +972,6 @@ void parseAndVerifyInsertUpdatePayload(std::vector<EDCServerPayloadInfo>* pField void collectEDCServerInfo(std::vector<EDCServerPayloadInfo>* pFields, ConstDataRange cdr, - StringData fieldPath) { // TODO - validate field is actually indexed in the schema? @@ -2053,7 +2052,8 @@ ESCDerivedFromDataTokenAndContentionFactorToken EDCServerPayloadInfo::getESCToke } void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, - const EncryptedFieldConfig& efc) { + const EncryptedFieldConfig& efc, + bool bypassDocumentValidation) { stdx::unordered_set<std::string> indexedFields; for (auto f : efc.getFields()) { if (f.getQueries().has_value()) { @@ -2070,6 +2070,11 @@ void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, indexedFields.contains(fieldPath.toString())); } }); + + // We should ensure that the user is not manually modifying the safe content array. + uassert(6666200, + str::stream() << "Cannot modify " << kSafeContent << " field in document.", + !obj.hasField(kSafeContent) || bypassDocumentValidation); } diff --git a/src/mongo/crypto/fle_crypto.h b/src/mongo/crypto/fle_crypto.h index 2520b7a02b3..5feac8ca2d3 100644 --- a/src/mongo/crypto/fle_crypto.h +++ b/src/mongo/crypto/fle_crypto.h @@ -1010,7 +1010,9 @@ public: /** * Validate that payload is compatible with schema */ - static void validateEncryptedFieldInfo(BSONObj& obj, const EncryptedFieldConfig& efc); + static void validateEncryptedFieldInfo(BSONObj& obj, + const EncryptedFieldConfig& efc, + bool bypassDocumentValidation); /** * Get information about all FLE2InsertUpdatePayload payloads diff --git a/src/mongo/crypto/fle_crypto_test.cpp b/src/mongo/crypto/fle_crypto_test.cpp index 99b470497cb..4c4355ebb9f 100644 --- a/src/mongo/crypto/fle_crypto_test.cpp +++ b/src/mongo/crypto/fle_crypto_test.cpp @@ -728,7 +728,7 @@ BSONObj encryptDocument(BSONObj obj, auto result = FLEClientCrypto::transformPlaceholders(obj, keyVault); if (nullptr != efc) { - EDCServerCollection::validateEncryptedFieldInfo(result, *efc); + EDCServerCollection::validateEncryptedFieldInfo(result, *efc, false); } // Start Server Side diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 29e343e44bc..b79c8c78914 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -38,6 +38,7 @@ #include "mongo/bson/ordering.h" #include "mongo/bson/simple_bsonelement_comparator.h" #include "mongo/bson/simple_bsonobj_comparator.h" +#include "mongo/crypto/fle_crypto.h" #include "mongo/db/auth/security_token.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_options.h" @@ -816,7 +817,6 @@ Status CollectionImpl::insertDocumentsForOplog(OperationContext* opCtx, return status; } - Status CollectionImpl::insertDocuments(OperationContext* opCtx, const std::vector<InsertStatement>::const_iterator begin, const std::vector<InsertStatement>::const_iterator end, @@ -840,8 +840,20 @@ Status CollectionImpl::insertDocuments(OperationContext* opCtx, } auto status = _checkValidationAndParseResult(opCtx, it->doc); - if (!status.isOK()) + if (!status.isOK()) { return status; + } + + auto& validationSettings = DocumentValidationSettings::get(opCtx); + + if (getCollectionOptions().encryptedFieldConfig && + !validationSettings.isSchemaValidationDisabled() && + !validationSettings.isSafeContentValidationDisabled() && + it->doc.hasField(kSafeContent)) { + return Status(ErrorCodes::BadValue, + str::stream() + << "Cannot insert a document with field name " << kSafeContent); + } } const SnapshotId sid = opCtx->recoveryUnit()->getSnapshotId(); @@ -1342,6 +1354,17 @@ void CollectionImpl::deleteDocument(OperationContext* opCtx, } } +bool compareSafeContentElem(const BSONObj& oldDoc, const BSONObj& newDoc) { + if (newDoc.hasField(kSafeContent) != oldDoc.hasField(kSafeContent)) { + return false; + } + if (!newDoc.hasField(kSafeContent)) { + return true; + } + + return newDoc.getField(kSafeContent).binaryEqual(oldDoc.getField(kSafeContent)); +} + RecordId CollectionImpl::updateDocument(OperationContext* opCtx, RecordId oldLocation, const Snapshotted<BSONObj>& oldDoc, @@ -1366,6 +1389,17 @@ RecordId CollectionImpl::updateDocument(OperationContext* opCtx, } } + auto& validationSettings = DocumentValidationSettings::get(opCtx); + if (getCollectionOptions().encryptedFieldConfig && + !validationSettings.isSchemaValidationDisabled() && + !validationSettings.isSafeContentValidationDisabled()) { + + uassert(ErrorCodes::BadValue, + str::stream() << "New document and old document both need to have " << kSafeContent + << " field.", + compareSafeContentElem(oldDoc.value(), newDoc)); + } + dassert(opCtx->lockState()->isCollectionLockedForMode(ns(), MODE_IX)); invariant(oldDoc.snapshotId() == opCtx->recoveryUnit()->getSnapshotId()); invariant(newDoc.isOwned()); diff --git a/src/mongo/db/catalog/document_validation.h b/src/mongo/db/catalog/document_validation.h index 875f255c565..47db304d79d 100644 --- a/src/mongo/db/catalog/document_validation.h +++ b/src/mongo/db/catalog/document_validation.h @@ -52,7 +52,7 @@ class DocumentValidationSettings { public: enum flag : std::uint8_t { /* - * Enables document validation (both schema and internal). + * Enables document validation (schema, internal, and safeContent). */ kEnableValidation = 0x00, /* @@ -67,6 +67,12 @@ public: * doesn't comply with internal validation rules. */ kDisableInternalValidation = 0x02, + /* + * If set, modifications to the safeContent array are allowed. This flag is only + * enabled when bypass document validation is enabled or if crudProcessed is true + * in the query. + */ + kDisableSafeContentValidation = 0x04, }; using Flags = std::uint8_t; @@ -92,6 +98,10 @@ public: return _flags & kDisableInternalValidation; } + bool isSafeContentValidationDisabled() const { + return _flags & kDisableSafeContentValidation; + } + bool isDocumentValidationEnabled() const { return _flags == kEnableValidation; } @@ -134,11 +144,29 @@ class DisableDocumentSchemaValidationIfTrue { public: DisableDocumentSchemaValidationIfTrue(OperationContext* opCtx, bool shouldDisableSchemaValidation) { - if (shouldDisableSchemaValidation) - _documentSchemaValidationDisabler.emplace(opCtx); + if (shouldDisableSchemaValidation) { + _documentSchemaValidationDisabler.emplace( + opCtx, DocumentValidationSettings::kDisableSchemaValidation); + } + } + +private: + boost::optional<DisableDocumentValidation> _documentSchemaValidationDisabler; +}; + +class DisableSafeContentValidationIfTrue { +public: + DisableSafeContentValidationIfTrue(OperationContext* opCtx, + bool shouldDisableSchemaValidation, + bool encryptionInformationCrudProcessed) { + if (shouldDisableSchemaValidation || encryptionInformationCrudProcessed) { + _documentSchemaValidationDisabler.emplace( + opCtx, DocumentValidationSettings::kDisableSafeContentValidation); + } } private: boost::optional<DisableDocumentValidation> _documentSchemaValidationDisabler; }; + } // namespace mongo diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index 00203a4c485..abbc0d834fd 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -636,10 +636,15 @@ write_ops::FindAndModifyCommandReply CmdFindAndModify::Invocation::typedRun( // Collect metrics. CmdFindAndModify::collectMetrics(req); - boost::optional<DisableDocumentValidation> maybeDisableValidation; - if (req.getBypassDocumentValidation().value_or(false)) { - maybeDisableValidation.emplace(opCtx); - } + auto disableDocumentValidation = req.getBypassDocumentValidation().value_or(false); + auto fleCrudProcessed = + write_ops_exec::getFleCrudProcessed(opCtx, req.getEncryptionInformation()); + + DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, + disableDocumentValidation); + + DisableSafeContentValidationIfTrue safeContentValidationDisabler( + opCtx, disableDocumentValidation, fleCrudProcessed); const auto inTransaction = opCtx->inMultiDocumentTransaction(); uassert(50781, diff --git a/src/mongo/db/commands/fle_compact_test.cpp b/src/mongo/db/commands/fle_compact_test.cpp index 18c52f548ef..26153aadcc8 100644 --- a/src/mongo/db/commands/fle_compact_test.cpp +++ b/src/mongo/db/commands/fle_compact_test.cpp @@ -395,8 +395,13 @@ void FleCompactTest::doSingleInsert(int id, BSONObj encryptedFieldsObj) { auto efc = generateEncryptedFieldConfig(encryptedFieldsObj.getFieldNames<std::set<std::string>>()); - uassertStatusOK(processInsert( - _queryImpl.get(), _namespaces.edcNss, serverPayload, efc, kUninitializedTxnNumber, result)); + uassertStatusOK(processInsert(_queryImpl.get(), + _namespaces.edcNss, + serverPayload, + efc, + kUninitializedTxnNumber, + result, + false)); } void FleCompactTest::doSingleDelete(int id, BSONObj encryptedFieldsObj) { diff --git a/src/mongo/db/commands/write_commands.cpp b/src/mongo/db/commands/write_commands.cpp index 9e6d189b4a3..0254baca47d 100644 --- a/src/mongo/db/commands/write_commands.cpp +++ b/src/mongo/db/commands/write_commands.cpp @@ -529,7 +529,8 @@ public: write_ops::InsertCommandReply typedRun(OperationContext* opCtx) final try { transactionChecks(opCtx, ns()); - if (request().getEncryptionInformation().has_value()) { + if (request().getEncryptionInformation().has_value() && + !request().getEncryptionInformation()->getCrudProcessed()) { write_ops::InsertCommandReply insertReply; auto batch = processFLEInsert(opCtx, request(), &insertReply); if (batch == FLEBatchResult::kProcessed) { @@ -1456,7 +1457,8 @@ public: write_ops::UpdateCommandReply updateReply; OperationSource source = OperationSource::kStandard; - if (request().getEncryptionInformation().has_value()) { + if (request().getEncryptionInformation().has_value() && + !request().getEncryptionInformation().get().getCrudProcessed()) { return processFLEUpdate(opCtx, request()); } diff --git a/src/mongo/db/fle_crud.cpp b/src/mongo/db/fle_crud.cpp index 15b70d34cbe..58136f31c44 100644 --- a/src/mongo/db/fle_crud.cpp +++ b/src/mongo/db/fle_crud.cpp @@ -190,16 +190,20 @@ std::pair<FLEBatchResult, write_ops::InsertCommandReply> processInsert( auto edcNss = insertRequest.getNamespace(); auto ei = insertRequest.getEncryptionInformation().get(); + bool bypassDocumentValidation = + insertRequest.getWriteCommandRequestBase().getBypassDocumentValidation(); + auto efc = EncryptionInformationHelpers::getAndValidateSchema(edcNss, ei); auto documents = insertRequest.getDocuments(); // TODO - how to check if a document will be too large??? + uassert(6371202, "Only single insert batches are supported in Queryable Encryption", documents.size() == 1); auto document = documents[0]; - EDCServerCollection::validateEncryptedFieldInfo(document, efc); + EDCServerCollection::validateEncryptedFieldInfo(document, efc, bypassDocumentValidation); auto serverPayload = std::make_shared<std::vector<EDCServerPayloadInfo>>( EDCServerCollection::getEncryptedFieldInfo(document)); @@ -223,8 +227,8 @@ std::pair<FLEBatchResult, write_ops::InsertCommandReply> processInsert( auto swResult = trun->runNoThrow( opCtx, - [sharedInsertBlock, reply, ownedDocument](const txn_api::TransactionClient& txnClient, - ExecutorPtr txnExec) { + [sharedInsertBlock, reply, ownedDocument, bypassDocumentValidation]( + const txn_api::TransactionClient& txnClient, ExecutorPtr txnExec) { FLEQueryInterfaceImpl queryImpl(txnClient, getGlobalServiceContext()); auto [edcNss2, efc2, serverPayload2, stmtId2] = *sharedInsertBlock.get(); @@ -234,8 +238,13 @@ std::pair<FLEBatchResult, write_ops::InsertCommandReply> processInsert( fleCrudHangPreInsert.pauseWhileSet(); } - *reply = uassertStatusOK(processInsert( - &queryImpl, edcNss2, *serverPayload2.get(), efc2, stmtId2, ownedDocument)); + *reply = uassertStatusOK(processInsert(&queryImpl, + edcNss2, + *serverPayload2.get(), + efc2, + stmtId2, + ownedDocument, + bypassDocumentValidation)); if (MONGO_unlikely(fleCrudHangInsert.shouldFail())) { LOGV2(6371903, "Hanging due to fleCrudHangInsert fail point"); @@ -441,7 +450,8 @@ void processFieldsForInsert(FLEQueryInterface* queryImpl, const NamespaceString& edcNss, std::vector<EDCServerPayloadInfo>& serverPayload, const EncryptedFieldConfig& efc, - int32_t* pStmtId) { + int32_t* pStmtId, + bool bypassDocumentValidation) { NamespaceString nssEsc(edcNss.db(), efc.getEscCollection().get()); @@ -509,7 +519,8 @@ void processFieldsForInsert(FLEQueryInterface* queryImpl, ECOCCollection::generateDocument(payload.fieldPathName, payload.payload.getEncryptedTokens()), pStmtId, - false)); + false, + bypassDocumentValidation)); checkWriteErrors(ecocInsertReply); } } @@ -719,9 +730,11 @@ StatusWith<write_ops::InsertCommandReply> processInsert( std::vector<EDCServerPayloadInfo>& serverPayload, const EncryptedFieldConfig& efc, int32_t stmtId, - BSONObj document) { + BSONObj document, + bool bypassDocumentValidation) { - processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); + processFieldsForInsert( + queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); auto finalDoc = EDCServerCollection::finalizeForInsert(document, serverPayload); @@ -792,6 +805,9 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, auto tokenMap = EncryptionInformationHelpers::getDeleteTokens(edcNss, ei); const auto updateOpEntry = updateRequest.getUpdates()[0]; + auto bypassDocumentValidation = + updateRequest.getWriteCommandRequestBase().getBypassDocumentValidation(); + const auto updateModification = updateOpEntry.getU(); int32_t stmtId = getStmtIdForWriteAt(updateRequest, 0); @@ -814,10 +830,11 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, if (updateModification.type() == write_ops::UpdateModification::Type::kModifier) { auto updateModifier = updateModification.getUpdateModifier(); auto setObject = updateModifier.getObjectField("$set"); - EDCServerCollection::validateEncryptedFieldInfo(setObject, efc); + EDCServerCollection::validateEncryptedFieldInfo(setObject, efc, bypassDocumentValidation); serverPayload = EDCServerCollection::getEncryptedFieldInfo(updateModifier); - processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); + processFieldsForInsert( + queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); // Step 2 ---- auto pushUpdate = EDCServerCollection::finalizeForUpdate(updateModifier, serverPayload); @@ -826,10 +843,12 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, pushUpdate, write_ops::UpdateModification::ClassicTag(), false)); } else { auto replacementDocument = updateModification.getUpdateReplacement(); - EDCServerCollection::validateEncryptedFieldInfo(replacementDocument, efc); + EDCServerCollection::validateEncryptedFieldInfo( + replacementDocument, efc, bypassDocumentValidation); serverPayload = EDCServerCollection::getEncryptedFieldInfo(replacementDocument); - processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); + processFieldsForInsert( + queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); // Step 2 ---- auto safeContentReplace = @@ -844,6 +863,8 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, newUpdateRequest.setUpdates({newUpdateOpEntry}); newUpdateRequest.getWriteCommandRequestBase().setStmtIds(boost::none); newUpdateRequest.getWriteCommandRequestBase().setStmtId(stmtId); + newUpdateRequest.getWriteCommandRequestBase().setBypassDocumentValidation( + bypassDocumentValidation); ++stmtId; auto [updateReply, originalDocument] = @@ -901,6 +922,10 @@ FLEBatchResult processFLEBatch(OperationContext* opCtx, BatchedCommandResponse* response, boost::optional<OID> targetEpoch) { + if (request.getWriteCommandRequestBase().getEncryptionInformation()->getCrudProcessed()) { + return FLEBatchResult::kNotProcessed; + } + // TODO (SERVER-65077): Remove FCV check once 6.0 is released uassert(6371209, "Queryable Encryption is only supported when FCV supports 6.0", @@ -1020,6 +1045,9 @@ write_ops::FindAndModifyCommandReply processFindAndModify( auto newFindAndModifyRequest = findAndModifyRequest; + const auto bypassDocumentValidation = + findAndModifyRequest.getBypassDocumentValidation().value_or(false); + // Step 0 ---- // Rewrite filter auto highCardinalityModeAllowed = findAndModifyRequest.getUpsert().value_or(false) @@ -1049,9 +1077,11 @@ write_ops::FindAndModifyCommandReply processFindAndModify( if (updateModification.type() == write_ops::UpdateModification::Type::kModifier) { auto updateModifier = updateModification.getUpdateModifier(); auto setObject = updateModifier.getObjectField("$set"); - EDCServerCollection::validateEncryptedFieldInfo(setObject, efc); + EDCServerCollection::validateEncryptedFieldInfo( + setObject, efc, bypassDocumentValidation); serverPayload = EDCServerCollection::getEncryptedFieldInfo(updateModifier); - processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); + processFieldsForInsert( + queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); auto pushUpdate = EDCServerCollection::finalizeForUpdate(updateModifier, serverPayload); @@ -1060,10 +1090,12 @@ write_ops::FindAndModifyCommandReply processFindAndModify( pushUpdate, write_ops::UpdateModification::ClassicTag(), false); } else { auto replacementDocument = updateModification.getUpdateReplacement(); - EDCServerCollection::validateEncryptedFieldInfo(replacementDocument, efc); + EDCServerCollection::validateEncryptedFieldInfo( + replacementDocument, efc, bypassDocumentValidation); serverPayload = EDCServerCollection::getEncryptedFieldInfo(replacementDocument); - processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); + processFieldsForInsert( + queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); // Step 2 ---- auto safeContentReplace = @@ -1267,10 +1299,23 @@ uint64_t FLEQueryInterfaceImpl::countDocuments(const NamespaceString& nss) { } StatusWith<write_ops::InsertCommandReply> FLEQueryInterfaceImpl::insertDocument( - const NamespaceString& nss, BSONObj obj, StmtId* pStmtId, bool translateDuplicateKey) { + const NamespaceString& nss, + BSONObj obj, + StmtId* pStmtId, + bool translateDuplicateKey, + bool bypassDocumentValidation) { write_ops::InsertCommandRequest insertRequest(nss); insertRequest.setDocuments({obj}); + EncryptionInformation encryptionInformation; + encryptionInformation.setCrudProcessed(true); + + // We need to set an empty BSON object here for the schema. + encryptionInformation.setSchema(BSONObj()); + insertRequest.getWriteCommandRequestBase().setEncryptionInformation(encryptionInformation); + insertRequest.getWriteCommandRequestBase().setBypassDocumentValidation( + bypassDocumentValidation); + int32_t stmtId = *pStmtId; if (stmtId != kUninitializedStmtId) { (*pStmtId)++; @@ -1355,6 +1400,7 @@ std::pair<write_ops::UpdateCommandReply, BSONObj> FLEQueryInterfaceImpl::updateW findAndModifyRequest.setLet( mergeLetAndCVariables(updateRequest.getLet(), updateOpEntry.getC())); findAndModifyRequest.setStmtId(updateRequest.getStmtId()); + findAndModifyRequest.setBypassDocumentValidation(updateRequest.getBypassDocumentValidation()); auto ei2 = ei; ei2.setCrudProcessed(true); @@ -1396,9 +1442,15 @@ std::pair<write_ops::UpdateCommandReply, BSONObj> FLEQueryInterfaceImpl::updateW } write_ops::UpdateCommandReply FLEQueryInterfaceImpl::update( - const NamespaceString& nss, - int32_t stmtId, - const write_ops::UpdateCommandRequest& updateRequest) { + const NamespaceString& nss, int32_t stmtId, write_ops::UpdateCommandRequest& updateRequest) { + + invariant(!updateRequest.getWriteCommandRequestBase().getEncryptionInformation()); + + EncryptionInformation encryptionInformation; + encryptionInformation.setCrudProcessed(true); + + encryptionInformation.setSchema(BSONObj()); + updateRequest.getWriteCommandRequestBase().setEncryptionInformation(encryptionInformation); dassert(updateRequest.getStmtIds().value_or(std::vector<int32_t>()).empty()); diff --git a/src/mongo/db/fle_crud.h b/src/mongo/db/fle_crud.h index 738e85b8996..7c8d93ae1f9 100644 --- a/src/mongo/db/fle_crud.h +++ b/src/mongo/db/fle_crud.h @@ -261,7 +261,11 @@ public: * FLEStateCollectionContention instead. */ virtual StatusWith<write_ops::InsertCommandReply> insertDocument( - const NamespaceString& nss, BSONObj obj, StmtId* pStmtId, bool translateDuplicateKey) = 0; + const NamespaceString& nss, + BSONObj obj, + StmtId* pStmtId, + bool translateDuplicateKey, + bool bypassDocumentValidation = false) = 0; /** * Delete a single document with the given query. @@ -294,7 +298,7 @@ public: virtual write_ops::UpdateCommandReply update( const NamespaceString& nss, int32_t stmtId, - const write_ops::UpdateCommandRequest& updateRequest) = 0; + write_ops::UpdateCommandRequest& updateRequest) = 0; /** * Do a single findAndModify request. @@ -325,10 +329,12 @@ public: uint64_t countDocuments(const NamespaceString& nss) final; - StatusWith<write_ops::InsertCommandReply> insertDocument(const NamespaceString& nss, - BSONObj obj, - int32_t* pStmtId, - bool translateDuplicateKey) final; + StatusWith<write_ops::InsertCommandReply> insertDocument( + const NamespaceString& nss, + BSONObj obj, + int32_t* pStmtId, + bool translateDuplicateKey, + bool bypassDocumentValidation = false) final; std::pair<write_ops::DeleteCommandReply, BSONObj> deleteWithPreimage( const NamespaceString& nss, @@ -340,10 +346,9 @@ public: const EncryptionInformation& ei, const write_ops::UpdateCommandRequest& updateRequest) final; - write_ops::UpdateCommandReply update( - const NamespaceString& nss, - int32_t stmtId, - const write_ops::UpdateCommandRequest& updateRequest) final; + write_ops::UpdateCommandReply update(const NamespaceString& nss, + int32_t stmtId, + write_ops::UpdateCommandRequest& updateRequest) final; write_ops::FindAndModifyCommandReply findAndModify( const NamespaceString& nss, @@ -408,7 +413,8 @@ StatusWith<write_ops::InsertCommandReply> processInsert( std::vector<EDCServerPayloadInfo>& serverPayload, const EncryptedFieldConfig& efc, int32_t stmtId, - BSONObj document); + BSONObj document, + bool bypassDocumentValidation = false); /** * Process a FLE delete with the query interface diff --git a/src/mongo/db/fle_crud_test.cpp b/src/mongo/db/fle_crud_test.cpp index d97f4621987..0a5d7dfc37c 100644 --- a/src/mongo/db/fle_crud_test.cpp +++ b/src/mongo/db/fle_crud_test.cpp @@ -154,8 +154,12 @@ protected: void assertDocumentCounts(uint64_t edc, uint64_t esc, uint64_t ecc, uint64_t ecoc); - void doSingleInsert(int id, BSONElement element); - void doSingleInsert(int id, BSONObj obj); + void testValidateEncryptedFieldInfo(BSONObj obj, bool bypassValidation); + + void testValidateTags(BSONObj obj); + + void doSingleInsert(int id, BSONElement element, bool bypassDocumentValidation = false); + void doSingleInsert(int id, BSONObj obj, bool bypassDocumentValidation = false); void doSingleInsertWithContention( int id, BSONElement element, int64_t cm, uint64_t cf, EncryptedFieldConfig efc); @@ -407,7 +411,7 @@ void FleCrudTest::doSingleWideInsert(int id, uint64_t fieldCount, ValueGenerator auto efc = getTestEncryptedFieldConfig(); - uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result)); + uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result, false)); } @@ -452,7 +456,16 @@ std::vector<char> generateSinglePlaceholder(BSONElement value, int64_t cm = 0) { return v; } -void FleCrudTest::doSingleInsert(int id, BSONElement element) { +void FleCrudTest::testValidateEncryptedFieldInfo(BSONObj obj, bool bypassValidation) { + auto efc = getTestEncryptedFieldConfig(); + EDCServerCollection::validateEncryptedFieldInfo(obj, efc, bypassValidation); +} + +void FleCrudTest::testValidateTags(BSONObj obj) { + FLEClientCrypto::validateTagsArray(obj); +} + +void FleCrudTest::doSingleInsert(int id, BSONElement element, bool bypassDocumentValidation) { auto buf = generateSinglePlaceholder(element); BSONObjBuilder builder; builder.append("_id", id); @@ -468,10 +481,10 @@ void FleCrudTest::doSingleInsert(int id, BSONElement element) { auto efc = getTestEncryptedFieldConfig(); - uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result)); + uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result, false)); } -void FleCrudTest::doSingleInsert(int id, BSONObj obj) { +void FleCrudTest::doSingleInsert(int id, BSONObj obj, bool bypassDocumentValidation) { doSingleInsert(id, obj.firstElement()); } @@ -491,7 +504,7 @@ void FleCrudTest::doSingleInsertWithContention( auto serverPayload = EDCServerCollection::getEncryptedFieldInfo(result); - uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result)); + uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result, false)); } void FleCrudTest::doSingleInsertWithContention( @@ -891,7 +904,6 @@ TEST_F(FleCrudTest, UpdateOneSameValue) { << "secret")); } - // Update one document with replacement TEST_F(FleCrudTest, UpdateOneReplace) { @@ -957,7 +969,16 @@ TEST_F(FleCrudTest, SetSafeContent) { builder.append("$set", BSON(kSafeContent << "foo")); auto result = builder.obj(); - ASSERT_THROWS_CODE(doSingleUpdateWithUpdateDoc(1, result), DBException, 6371507); + ASSERT_THROWS_CODE(doSingleUpdateWithUpdateDoc(1, result), DBException, 6666200); +} + +// Test that EDCServerCollection::validateEncryptedFieldInfo checks that the +// safeContent cannot be present in the BSON obj. +TEST_F(FleCrudTest, testValidateEncryptedFieldConfig) { + testValidateEncryptedFieldInfo(BSON(kSafeContent << "secret"), true); + ASSERT_THROWS_CODE(testValidateEncryptedFieldInfo(BSON(kSafeContent << "secret"), false), + DBException, + 6666200); } // Update one document via findAndModify @@ -1039,6 +1060,11 @@ TEST_F(FleCrudTest, FindAndModify_RenameSafeContent) { ASSERT_THROWS_CODE(doFindAndModify(req), DBException, 6371506); } +TEST_F(FleCrudTest, validateTagsTest) { + testValidateTags(BSON(kSafeContent << BSON_ARRAY(123))); + ASSERT_THROWS_CODE(testValidateTags(BSON(kSafeContent << "foo")), DBException, 6371507); +} + // Mess with __safeContent__ and ensure the update errors TEST_F(FleCrudTest, FindAndModify_SetSafeContent) { doSingleInsert(1, @@ -1057,8 +1083,7 @@ TEST_F(FleCrudTest, FindAndModify_SetSafeContent) { req.setUpdate( write_ops::UpdateModification(result, write_ops::UpdateModification::ClassicTag{}, false)); - - ASSERT_THROWS_CODE(doFindAndModify(req), DBException, 6371507); + ASSERT_THROWS_CODE(doFindAndModify(req), DBException, 6666200); } TEST_F(FleTagsTest, InsertOne) { diff --git a/src/mongo/db/fle_query_interface_mock.cpp b/src/mongo/db/fle_query_interface_mock.cpp index 2aeb39788dd..b5ca4e1e9cd 100644 --- a/src/mongo/db/fle_query_interface_mock.cpp +++ b/src/mongo/db/fle_query_interface_mock.cpp @@ -54,7 +54,11 @@ uint64_t FLEQueryInterfaceMock::countDocuments(const NamespaceString& nss) { } StatusWith<write_ops::InsertCommandReply> FLEQueryInterfaceMock::insertDocument( - const NamespaceString& nss, BSONObj obj, StmtId* pStmtId, bool translateDuplicateKey) { + const NamespaceString& nss, + BSONObj obj, + StmtId* pStmtId, + bool translateDuplicateKey, + bool bypassDocumentValidation) { repl::TimestampedBSONObj tb; tb.obj = obj; @@ -132,9 +136,7 @@ std::pair<write_ops::UpdateCommandReply, BSONObj> FLEQueryInterfaceMock::updateW } write_ops::UpdateCommandReply FLEQueryInterfaceMock::update( - const NamespaceString& nss, - int32_t stmtId, - const write_ops::UpdateCommandRequest& updateRequest) { + const NamespaceString& nss, int32_t stmtId, write_ops::UpdateCommandRequest& updateRequest) { auto [reply, _] = updateWithPreimage(nss, EncryptionInformation(), updateRequest); return reply; } diff --git a/src/mongo/db/fle_query_interface_mock.h b/src/mongo/db/fle_query_interface_mock.h index 229d2c08dfe..a89fc71ce1e 100644 --- a/src/mongo/db/fle_query_interface_mock.h +++ b/src/mongo/db/fle_query_interface_mock.h @@ -47,10 +47,12 @@ public: uint64_t countDocuments(const NamespaceString& nss) final; - StatusWith<write_ops::InsertCommandReply> insertDocument(const NamespaceString& nss, - BSONObj obj, - StmtId* pStmtId, - bool translateDuplicateKey) final; + StatusWith<write_ops::InsertCommandReply> insertDocument( + const NamespaceString& nss, + BSONObj obj, + StmtId* pStmtId, + bool translateDuplicateKey, + bool bypassDocumentValidation = false) final; std::pair<write_ops::DeleteCommandReply, BSONObj> deleteWithPreimage( const NamespaceString& nss, @@ -62,10 +64,9 @@ public: const EncryptionInformation& ei, const write_ops::UpdateCommandRequest& updateRequest) final; - write_ops::UpdateCommandReply update( - const NamespaceString& nss, - int32_t stmtId, - const write_ops::UpdateCommandRequest& updateRequest) final; + write_ops::UpdateCommandReply update(const NamespaceString& nss, + int32_t stmtId, + write_ops::UpdateCommandRequest& updateRequest) final; write_ops::FindAndModifyCommandReply findAndModify( const NamespaceString& nss, diff --git a/src/mongo/db/ops/write_ops_exec.cpp b/src/mongo/db/ops/write_ops_exec.cpp index 6fd7e2200c7..f7fc2a84efd 100644 --- a/src/mongo/db/ops/write_ops_exec.cpp +++ b/src/mongo/db/ops/write_ops_exec.cpp @@ -602,11 +602,36 @@ SingleWriteResult makeWriteResultForInsertOrDeleteRetry() { return res; } + +// Returns the flags that determine the type of document validation we want to +// perform. First item in the tuple determines whether to bypass document validation altogether, +// second item determines if _safeContent_ array can be modified in an encrypted collection. +std::tuple<bool, bool> getDocumentValidationFlags(OperationContext* opCtx, + const write_ops::WriteCommandRequestBase& req) { + auto& encryptionInfo = req.getEncryptionInformation(); + const bool fleCrudProcessed = getFleCrudProcessed(opCtx, encryptionInfo); + return std::make_tuple(req.getBypassDocumentValidation(), fleCrudProcessed); +} } // namespace +bool getFleCrudProcessed(OperationContext* opCtx, + const boost::optional<EncryptionInformation>& encryptionInfo) { + if (encryptionInfo && encryptionInfo->getCrudProcessed().value_or(false)) { + uassert(6666201, + "External users cannot have crudProcessed enabled", + AuthorizationSession::get(opCtx->getClient()) + ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), + ActionType::internal)); + + return true; + } + return false; +} + WriteResult performInserts(OperationContext* opCtx, const write_ops::InsertCommandRequest& wholeOp, OperationSource source) { + // Insert performs its own retries, so we should only be within a WriteUnitOfWork when run in a // transaction. auto txnParticipant = TransactionParticipant::get(opCtx); @@ -641,8 +666,15 @@ WriteResult performInserts(OperationContext* opCtx, uassertStatusOK(userAllowedWriteNS(opCtx, wholeOp.getNamespace())); } - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler( - opCtx, wholeOp.getWriteCommandRequestBase().getBypassDocumentValidation()); + const auto [disableDocumentValidation, fleCrudProcessed] = + getDocumentValidationFlags(opCtx, wholeOp.getWriteCommandRequestBase()); + + DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, + disableDocumentValidation); + + DisableSafeContentValidationIfTrue safeContentValidationDisabler( + opCtx, disableDocumentValidation, fleCrudProcessed); + LastOpFixer lastOpFixer(opCtx, wholeOp.getNamespace()); WriteResult out; @@ -1000,8 +1032,15 @@ WriteResult performUpdates(OperationContext* opCtx, (txnParticipant && opCtx->inMultiDocumentTransaction())); uassertStatusOK(userAllowedWriteNS(opCtx, ns)); - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler( - opCtx, wholeOp.getWriteCommandRequestBase().getBypassDocumentValidation()); + const auto [disableDocumentValidation, fleCrudProcessed] = + getDocumentValidationFlags(opCtx, wholeOp.getWriteCommandRequestBase()); + + DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, + disableDocumentValidation); + + DisableSafeContentValidationIfTrue safeContentValidationDisabler( + opCtx, disableDocumentValidation, fleCrudProcessed); + LastOpFixer lastOpFixer(opCtx, ns); bool containsRetry = false; @@ -1227,8 +1266,15 @@ WriteResult performDeletes(OperationContext* opCtx, (txnParticipant && opCtx->inMultiDocumentTransaction())); uassertStatusOK(userAllowedWriteNS(opCtx, ns)); - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler( - opCtx, wholeOp.getWriteCommandRequestBase().getBypassDocumentValidation()); + const auto [disableDocumentValidation, fleCrudProcessed] = + getDocumentValidationFlags(opCtx, wholeOp.getWriteCommandRequestBase()); + + DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, + disableDocumentValidation); + + DisableSafeContentValidationIfTrue safeContentValidationDisabler( + opCtx, disableDocumentValidation, fleCrudProcessed); + LastOpFixer lastOpFixer(opCtx, ns); bool containsRetry = false; diff --git a/src/mongo/db/ops/write_ops_exec.h b/src/mongo/db/ops/write_ops_exec.h index 548a3034713..3550a51c1ce 100644 --- a/src/mongo/db/ops/write_ops_exec.h +++ b/src/mongo/db/ops/write_ops_exec.h @@ -64,6 +64,9 @@ struct WriteResult { bool canContinue = true; }; +bool getFleCrudProcessed(OperationContext* opCtx, + const boost::optional<EncryptionInformation>& encryptionInfo); + /** * Performs a batch of inserts, updates, or deletes. * diff --git a/src/mongo/idl/generic_argument.idl b/src/mongo/idl/generic_argument.idl index fdf340cbef1..5cb308bd2e1 100644 --- a/src/mongo/idl/generic_argument.idl +++ b/src/mongo/idl/generic_argument.idl @@ -104,6 +104,7 @@ generic_argument_lists: mayBypassWriteBlocking: forward_to_shards: true + generic_reply_field_lists: generic_reply_fields_api_v1: description: "Fields that may appear in any command reply. These are guaranteed backwards-compatible for as long as the server supports API Version 1." diff --git a/src/mongo/s/write_ops/batch_write_op.cpp b/src/mongo/s/write_ops/batch_write_op.cpp index b034bcd0ee4..a61ee3dd4bf 100644 --- a/src/mongo/s/write_ops/batch_write_op.cpp +++ b/src/mongo/s/write_ops/batch_write_op.cpp @@ -289,6 +289,13 @@ void populateCollectionUUIDMismatch(OperationContext* opCtx, } } +int getEncryptionInformationSize(const BatchedCommandRequest& req) { + if (!req.getWriteCommandRequestBase().getEncryptionInformation()) { + return 0; + } + return req.getWriteCommandRequestBase().getEncryptionInformation().get().toBSON().objsize(); +} + } // namespace BatchWriteOp::BatchWriteOp(OperationContext* opCtx, const BatchedCommandRequest& clientRequest) @@ -421,6 +428,7 @@ Status BatchWriteOp::targetBatch( // // The constant 4 is chosen as the size of the BSON representation of the stmtId. const int writeSizeBytes = getWriteSizeBytes(writeOp) + + getEncryptionInformationSize(_clientRequest) + write_ops::kWriteCommandBSONArrayPerElementOverheadBytes + (_batchTxnNum ? write_ops::kWriteCommandBSONArrayPerElementOverheadBytes + 4 : 0); @@ -583,6 +591,9 @@ BatchedCommandRequest BatchWriteOp::buildBatchRequest(const TargetedWriteBatch& wcb.setOrdered(_clientRequest.getWriteCommandRequestBase().getOrdered()); wcb.setCollectionUUID(_clientRequest.getWriteCommandRequestBase().getCollectionUUID()); + wcb.setEncryptionInformation( + _clientRequest.getWriteCommandRequestBase().getEncryptionInformation()); + if (targeter.isShardedTimeSeriesBucketsNamespace() && !_clientRequest.getNS().isTimeseriesBucketsCollection()) { wcb.setIsTimeseriesNamespace(true); |