diff options
author | Mark Benvenuto <mark.benvenuto@mongodb.com> | 2023-04-04 11:26:53 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-04-04 17:16:13 +0000 |
commit | 2d5b6fb51af722de35c34678c8d146743bc52190 (patch) | |
tree | 451194fd2cacc75dbe16d2d67ab0a1c345d29f16 /src/mongo | |
parent | 05794b52523f287f828a8fa692ac3f3ed6cd5dd1 (diff) | |
download | mongo-2d5b6fb51af722de35c34678c8d146743bc52190.tar.gz |
SERVER-75265 Improve queryable encryption handling in curop
Diffstat (limited to 'src/mongo')
28 files changed, 235 insertions, 79 deletions
diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp index 7375821c15b..db7a5253ef3 100644 --- a/src/mongo/db/clientcursor.cpp +++ b/src/mongo/db/clientcursor.cpp @@ -125,6 +125,8 @@ ClientCursor::ClientCursor(ClientCursorParams params, _planCacheKey(CurOp::get(operationUsingCursor)->debug().planCacheKey), _queryHash(CurOp::get(operationUsingCursor)->debug().queryHash), _telemetryStoreKey(CurOp::get(operationUsingCursor)->debug().telemetryStoreKey), + _shouldOmitDiagnosticInformation( + CurOp::get(operationUsingCursor)->debug().shouldOmitDiagnosticInformation), _opKey(operationUsingCursor->getOperationKey()) { invariant(_exec); invariant(_operationUsingCursor); diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index e4856e3854e..11e7b604ba0 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -313,6 +313,14 @@ public: _killPending = newValue; } + /** + * Returns true if operations with this cursor should be omitted from diagnostic sources such as + * currentOp and the profiler. + */ + bool shouldOmitDiagnosticInformation() const { + return _shouldOmitDiagnosticInformation; + } + private: friend class CursorManager; friend class ClientCursorPin; @@ -446,6 +454,9 @@ private: // Useful for diagnostics like telemetry. OpDebug::AdditiveMetrics _metrics; + // Flag to decide if diagnostic information should be omitted. + bool _shouldOmitDiagnosticInformation{false}; + // The client OperationKey associated with this cursor. boost::optional<OperationKey> _opKey; diff --git a/src/mongo/db/commands/count_cmd.cpp b/src/mongo/db/commands/count_cmd.cpp index a3829237d91..821ad1bc6de 100644 --- a/src/mongo/db/commands/count_cmd.cpp +++ b/src/mongo/db/commands/count_cmd.cpp @@ -164,7 +164,11 @@ public: } if (shouldDoFLERewrite(request)) { - processFLECountD(opCtx, nss, &request); + if (!request.getEncryptionInformation()->getCrudProcessed().value_or(false)) { + processFLECountD(opCtx, nss, &request); + } + + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; } if (ctx->getView()) { @@ -248,7 +252,10 @@ public: auto curOp = CurOp::get(opCtx); curOp->beginQueryPlanningTimer(); if (shouldDoFLERewrite(request)) { - processFLECountD(opCtx, nss, &request); + if (!request.getEncryptionInformation()->getCrudProcessed().value_or(false)) { + processFLECountD(opCtx, nss, &request); + } + curOp->debug().shouldOmitDiagnosticInformation = true; } if (request.getMirrored().value_or(false)) { diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index 09fdac0ae47..5b4f8d4b922 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -338,12 +338,15 @@ void CmdFindAndModify::Invocation::explain(OperationContext* opCtx, const BSONObj& cmdObj = request().toBSON(BSONObj() /* commandPassthroughFields */); auto requestAndMsg = [&]() { - if (request().getEncryptionInformation() && - !request().getEncryptionInformation()->getCrudProcessed().value_or(false)) { - return processFLEFindAndModifyExplainMongod(opCtx, request()); - } else { - return std::pair{request(), OpMsgRequest()}; + if (request().getEncryptionInformation()) { + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + + if (!request().getEncryptionInformation()->getCrudProcessed().value_or(false)) { + return processFLEFindAndModifyExplainMongod(opCtx, request()); + } } + + return std::pair{request(), OpMsgRequest()}; }(); auto request = requestAndMsg.first; diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index 9b30e4fc5cf..206ead18516 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -790,6 +790,7 @@ public: // Rewrite any FLE find payloads that exist in the query if this is a FLE 2 query. if (shouldDoFLERewrite(findCommand)) { invariant(findCommand->getNamespaceOrUUID().nss()); + if (!findCommand->getEncryptionInformation()->getCrudProcessed().value_or(false)) { processFLEFindD( opCtx, findCommand->getNamespaceOrUUID().nss().value(), findCommand.get()); diff --git a/src/mongo/db/commands/fle2_compact_cmd.cpp b/src/mongo/db/commands/fle2_compact_cmd.cpp index 4e3d1845619..5fcbe58d727 100644 --- a/src/mongo/db/commands/fle2_compact_cmd.cpp +++ b/src/mongo/db/commands/fle2_compact_cmd.cpp @@ -64,6 +64,8 @@ Lock::ResourceMutex commandMutex("compactStructuredEncryptionDataCommandMutex"); CompactStats compactEncryptedCompactionCollection(OperationContext* opCtx, const CompactStructuredEncryptionData& request) { + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + uassert(6583201, str::stream() << CompactStructuredEncryptionData::kCommandName << " must be run through mongos in a sharded cluster", @@ -97,7 +99,6 @@ CompactStats compactEncryptedCompactionCollection(OperationContext* opCtx, } validateCompactRequest(request, *edc); - CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; auto namespaces = uassertStatusOK(EncryptedStateCollectionsNamespaces::createFromDataCollection(*edc)); @@ -274,6 +275,10 @@ public: bool adminOnly() const final { return false; } + + std::set<StringData> sensitiveFieldNames() const final { + return {CompactStructuredEncryptionData::kCompactionTokensFieldName}; + } } compactStructuredEncryptionDataCmd; } // namespace diff --git a/src/mongo/db/commands/fle2_get_count_info_command.cpp b/src/mongo/db/commands/fle2_get_count_info_command.cpp index 1c30c4bb4cc..0ad2d793e19 100644 --- a/src/mongo/db/commands/fle2_get_count_info_command.cpp +++ b/src/mongo/db/commands/fle2_get_count_info_command.cpp @@ -101,6 +101,8 @@ std::vector<QECountInfoReplyTokenSet> toGetTagRequestTupleSet( QECountInfosReply getTagsLocal(OperationContext* opCtx, const GetQueryableEncryptionCountInfo& request) { + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + uassert(741503, "FeatureFlagFLE2ProtocolVersion2 is not enabled", gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)); @@ -172,6 +174,10 @@ public: bool allowedInTransactions() const final { return true; } + + std::set<StringData> sensitiveFieldNames() const final { + return {GetQueryableEncryptionCountInfo::kTokensFieldName}; + } } getQueryableEncryptionCountInfoCmd; } // namespace diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp index 194f44a9092..8b2f980dd98 100644 --- a/src/mongo/db/commands/getmore_cmd.cpp +++ b/src/mongo/db/commands/getmore_cmd.cpp @@ -580,6 +580,8 @@ public: } curOp->debug().queryFramework = exec->getQueryFramework(); + curOp->debug().shouldOmitDiagnosticInformation = + cursorPin->shouldOmitDiagnosticInformation(); // Update the genericCursor stored in curOp with the new cursor stats. curOp->setGenericCursor_inlock(cursorPin->toGenericCursor()); diff --git a/src/mongo/db/commands/run_aggregate.cpp b/src/mongo/db/commands/run_aggregate.cpp index 6f503f0f850..c2f2701cd42 100644 --- a/src/mongo/db/commands/run_aggregate.cpp +++ b/src/mongo/db/commands/run_aggregate.cpp @@ -1012,10 +1012,13 @@ Status runAggregate(OperationContext* opCtx, // support querying against encrypted fields. if (shouldDoFLERewrite(request)) { CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; - // After this rewriting, the encryption info does not need to be kept around. - pipeline = processFLEPipelineD( - opCtx, nss, request.getEncryptionInformation().value(), std::move(pipeline)); - request.setEncryptionInformation(boost::none); + + if (!request.getEncryptionInformation()->getCrudProcessed().value_or(false)) { + pipeline = processFLEPipelineD( + opCtx, nss, request.getEncryptionInformation().value(), std::move(pipeline)); + request.getEncryptionInformation()->setCrudProcessed(true); + } + // Set the telemetryStoreKey to none so telemetry isn't collected when we've done a FLE // rewrite. CurOp::get(opCtx)->debug().telemetryStoreKey = boost::none; diff --git a/src/mongo/db/commands/write_commands.cpp b/src/mongo/db/commands/write_commands.cpp index b10383f968b..f8d958e929a 100644 --- a/src/mongo/db/commands/write_commands.cpp +++ b/src/mongo/db/commands/write_commands.cpp @@ -287,7 +287,8 @@ public: if (request().getEncryptionInformation().has_value()) { // Flag set here and in fle_crud.cpp since this only executes on a mongod. CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; - if (!request().getEncryptionInformation()->getCrudProcessed()) { + + if (!request().getEncryptionInformation()->getCrudProcessed().value_or(false)) { write_ops::InsertCommandReply insertReply; auto batch = processFLEInsert(opCtx, request(), &insertReply); if (batch == FLEBatchResult::kProcessed) { @@ -552,12 +553,17 @@ public: UpdateRequest updateRequest(request().getUpdates()[0]); updateRequest.setNamespaceString(request().getNamespace()); if (shouldDoFLERewrite(request())) { - updateRequest.setQuery( - processFLEWriteExplainD(opCtx, - write_ops::collationOf(request().getUpdates()[0]), - request(), - updateRequest.getQuery())); + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + + if (!request().getEncryptionInformation()->getCrudProcessed().value_or(false)) { + updateRequest.setQuery( + processFLEWriteExplainD(opCtx, + write_ops::collationOf(request().getUpdates()[0]), + request(), + updateRequest.getQuery())); + } } + updateRequest.setLegacyRuntimeConstants(request().getLegacyRuntimeConstants().value_or( Variables::generateRuntimeConstants(opCtx))); updateRequest.setLetParameters(request().getLet()); @@ -661,7 +667,10 @@ public: if (request().getEncryptionInformation().has_value()) { // Flag set here and in fle_crud.cpp since this only executes on a mongod. CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; - return processFLEDelete(opCtx, request()); + + if (!request().getEncryptionInformation()->getCrudProcessed().value_or(false)) { + return processFLEDelete(opCtx, request()); + } } if (isTimeseries(opCtx, request())) { @@ -730,8 +739,12 @@ public: const auto& firstDelete = request().getDeletes()[0]; BSONObj query = firstDelete.getQ(); if (shouldDoFLERewrite(request())) { - query = processFLEWriteExplainD( - opCtx, write_ops::collationOf(firstDelete), request(), query); + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + + if (!request().getEncryptionInformation()->getCrudProcessed().value_or(false)) { + query = processFLEWriteExplainD( + opCtx, write_ops::collationOf(firstDelete), request(), query); + } } deleteRequest.setQuery(std::move(query)); diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index 9327f52d95e..f2a33fe23a9 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -427,6 +427,18 @@ void CurOp::raiseDbProfileLevel(int dbProfileLevel) { static constexpr size_t appendMaxElementSize = 50 * 1024; +bool shouldOmitDiagnosticInformation(CurOp* curop) { + do { + if (curop->debug().shouldOmitDiagnosticInformation) { + return true; + } + + curop = curop->parent(); + } while (curop != nullptr); + + return false; +} + bool CurOp::completeAndLogOperation(logv2::LogComponent component, std::shared_ptr<const ProfileFilter> filter, boost::optional<size_t> responseLength, @@ -447,6 +459,11 @@ bool CurOp::completeAndLogOperation(logv2::LogComponent component, const auto executionTimeMillis = durationCount<Milliseconds>(*_debug.additiveMetrics.executionTime); + // Do not log the slow query information if asked to omit it + if (shouldOmitDiagnosticInformation(this)) { + return false; + } + if (_debug.isReplOplogGetMore) { oplogGetMoreStats.recordMillis(executionTimeMillis); } diff --git a/src/mongo/db/fle_crud.cpp b/src/mongo/db/fle_crud.cpp index a6beaf8ddd1..974605c8df9 100644 --- a/src/mongo/db/fle_crud.cpp +++ b/src/mongo/db/fle_crud.cpp @@ -39,6 +39,7 @@ #include "mongo/bson/bsontypes.h" #include "mongo/crypto/encryption_fields_gen.h" #include "mongo/crypto/fle_crypto.h" +#include "mongo/crypto/fle_field_schema_gen.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands/fle2_get_count_info_command_gen.h" #include "mongo/db/dbdirectclient.h" @@ -258,6 +259,20 @@ boost::intrusive_ptr<ExpressionContext> makeExpCtx(OperationContext* opCtx, return expCtx; } +/** + * We mark commands as "CrudProcessed" to ensure the various commands recognize them as QE related + * to ensure they are filtered out. + */ +EncryptionInformation makeEmptyProcessEncryptionInformation() { + EncryptionInformation encryptionInformation; + encryptionInformation.setCrudProcessed(true); + + // We need to set an empty BSON object here for the schema. + encryptionInformation.setSchema(BSONObj()); + + return encryptionInformation; +} + } // namespace using VTS = auth::ValidatedTenancyScope; @@ -1376,6 +1391,7 @@ FLEBatchResult processFLEBatch(OperationContext* opCtx, boost::optional<OID> targetEpoch) { CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + if (request.getWriteCommandRequestBase().getEncryptionInformation()->getCrudProcessed()) { return FLEBatchResult::kNotProcessed; } @@ -1458,7 +1474,9 @@ std::unique_ptr<BatchedCommandRequest> processFLEBatchExplain( &getTransactionWithRetriesForMongoS, fle::EncryptedCollScanModeAllowed::kAllow)); deleteRequest.setDeletes({newDeleteOp}); - deleteRequest.getWriteCommandRequestBase().setEncryptionInformation(boost::none); + deleteRequest.getWriteCommandRequestBase().setEncryptionInformation( + makeEmptyProcessEncryptionInformation()); + return std::make_unique<BatchedCommandRequest>(deleteRequest); } else if (request.getBatchType() == BatchedCommandRequest::BatchType_Update) { auto updateRequest = request.getUpdateRequest(); @@ -1475,7 +1493,8 @@ std::unique_ptr<BatchedCommandRequest> processFLEBatchExplain( &getTransactionWithRetriesForMongoS, encryptedCollScanModeAllowed)); updateRequest.setUpdates({newUpdateOp}); - updateRequest.getWriteCommandRequestBase().setEncryptionInformation(boost::none); + updateRequest.getWriteCommandRequestBase().setEncryptionInformation( + makeEmptyProcessEncryptionInformation()); return std::make_unique<BatchedCommandRequest>(updateRequest); } MONGO_UNREACHABLE; @@ -1718,7 +1737,7 @@ write_ops::FindAndModifyCommandRequest processFindAndModifyExplain( findAndModifyRequest.getQuery(), encryptedCollScanModeAllowed)); - newFindAndModifyRequest.setEncryptionInformation(boost::none); + newFindAndModifyRequest.setEncryptionInformation(makeEmptyProcessEncryptionInformation()); return newFindAndModifyRequest; } @@ -1734,6 +1753,8 @@ FLEBatchResult processFLEFindAndModify(OperationContext* opCtx, return FLEBatchResult::kNotProcessed; } + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + // FLE2 Mongos CRUD operations loopback through MongoS with EncryptionInformation as // findAndModify so query can do any necessary transformations. But on the nested call, CRUD // does not need to do any more work. @@ -1771,6 +1792,8 @@ BSONObj FLEQueryInterfaceImpl::getById(const NamespaceString& nss, BSONElement e find.setDollarTenant(tenantId); } + find.setEncryptionInformation(makeEmptyProcessEncryptionInformation()); + // Throws on error auto docs = _txnClient.exhaustiveFindSync(find); @@ -1857,12 +1880,9 @@ StatusWith<write_ops::InsertCommandReply> FLEQueryInterfaceImpl::insertDocuments if (tenantId && gMultitenancySupport) { insertRequest.setDollarTenant(tenantId); } - EncryptionInformation encryptionInformation; - encryptionInformation.setCrudProcessed(true); - // We need to set an empty BSON object here for the schema. - encryptionInformation.setSchema(BSONObj()); - insertRequest.getWriteCommandRequestBase().setEncryptionInformation(encryptionInformation); + insertRequest.getWriteCommandRequestBase().setEncryptionInformation( + makeEmptyProcessEncryptionInformation()); insertRequest.getWriteCommandRequestBase().setBypassDocumentValidation( bypassDocumentValidation); @@ -1942,8 +1962,10 @@ write_ops::DeleteCommandReply FLEQueryInterfaceImpl::deleteDocument( dassert(!deleteRequest.getWriteCommandRequestBase().getEncryptionInformation()); dassert(deleteRequest.getStmtIds().value_or(std::vector<int32_t>()).empty()); - auto response = _txnClient.runCRUDOpSync(BatchedCommandRequest(deleteRequest), {stmtId}); + deleteRequest.getWriteCommandRequestBase().setEncryptionInformation( + makeEmptyProcessEncryptionInformation()); + auto response = _txnClient.runCRUDOpSync(BatchedCommandRequest(deleteRequest), {stmtId}); write_ops::DeleteCommandReply reply; responseToReply(response, reply.getWriteCommandReplyBase()); return {reply}; @@ -2020,11 +2042,8 @@ write_ops::UpdateCommandReply FLEQueryInterfaceImpl::update( invariant(!updateRequest.getWriteCommandRequestBase().getEncryptionInformation()); - EncryptionInformation encryptionInformation; - encryptionInformation.setCrudProcessed(true); - - encryptionInformation.setSchema(BSONObj()); - updateRequest.getWriteCommandRequestBase().setEncryptionInformation(encryptionInformation); + updateRequest.getWriteCommandRequestBase().setEncryptionInformation( + makeEmptyProcessEncryptionInformation()); dassert(updateRequest.getStmtIds().value_or(std::vector<int32_t>()).empty()); @@ -2068,6 +2087,8 @@ std::vector<BSONObj> FLEQueryInterfaceImpl::findDocuments(const NamespaceString& } find.setFilter(filter); + find.setEncryptionInformation(makeEmptyProcessEncryptionInformation()); + // Throws on error return _txnClient.exhaustiveFindSync(find); } diff --git a/src/mongo/db/query/fle/server_rewrite.cpp b/src/mongo/db/query/fle/server_rewrite.cpp index 094fbb7ffdc..1360b761c18 100644 --- a/src/mongo/db/query/fle/server_rewrite.cpp +++ b/src/mongo/db/query/fle/server_rewrite.cpp @@ -346,11 +346,7 @@ void processFindCommand(OperationContext* opCtx, getTransaction, EncryptedCollScanModeAllowed::kAllow)); - EncryptionInformation encryptionInformation; - encryptionInformation.setCrudProcessed(true); - encryptionInformation.setSchema(BSONObj()); - - findCommand->setEncryptionInformation(encryptionInformation); + findCommand->getEncryptionInformation()->setCrudProcessed(true); } void processCountCommand(OperationContext* opCtx, @@ -371,11 +367,8 @@ void processCountCommand(OperationContext* opCtx, countCommand->getQuery().getOwned(), getTxn, EncryptedCollScanModeAllowed::kAllow)); - // The presence of encryptionInformation is a signal that this is a FLE request that requires - // special processing. Once we've rewritten the query, it's no longer a "special" FLE query, but - // a normal query that can be executed by the query system like any other, so remove - // encryptionInformation. - countCommand->setEncryptionInformation(boost::none); + + countCommand->getEncryptionInformation()->setCrudProcessed(true); } std::unique_ptr<Pipeline, PipelineDeleter> processPipeline( diff --git a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp index ed71048d293..b5473d9fd2f 100644 --- a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp +++ b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp @@ -73,6 +73,10 @@ public: return AllowedOnSecondary::kNever; } + std::set<StringData> sensitiveFieldNames() const final { + return {CompactStructuredEncryptionData::kCompactionTokensFieldName}; + } + class Invocation final : public InvocationBase { public: using InvocationBase::InvocationBase; @@ -80,6 +84,8 @@ public: Reply typedRun(OperationContext* opCtx) { bool useV1Protocol = false; + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + auto compactCoordinator = [&]() -> std::shared_ptr<ShardingDDLCoordinatorService::Instance> { FixedFCVRegion fixedFcvRegion(opCtx); diff --git a/src/mongo/s/commands/cluster_count_cmd.h b/src/mongo/s/commands/cluster_count_cmd.h index cb3ccef8bd2..1325c89a21e 100644 --- a/src/mongo/s/commands/cluster_count_cmd.h +++ b/src/mongo/s/commands/cluster_count_cmd.h @@ -115,8 +115,14 @@ public: std::vector<AsyncRequestsSender::Response> shardResponses; try { auto countRequest = CountCommandRequest::parse(IDLParserContext("count"), cmdObj); + if (shouldDoFLERewrite(countRequest)) { - processFLECountS(opCtx, nss, &countRequest); + + if (!countRequest.getEncryptionInformation()->getCrudProcessed().value_or(false)) { + processFLECountS(opCtx, nss, &countRequest); + } + + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; } // We only need to factor in the skip value when sending to the shards if we @@ -229,6 +235,8 @@ public: // If the command has encryptionInformation, rewrite the query as necessary. if (shouldDoFLERewrite(countRequest)) { processFLECountS(opCtx, nss, &countRequest); + + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; } BSONObj targetingQuery = countRequest.getQuery(); diff --git a/src/mongo/s/commands/cluster_find_and_modify_cmd.cpp b/src/mongo/s/commands/cluster_find_and_modify_cmd.cpp index 95822813511..b816c9f1278 100644 --- a/src/mongo/s/commands/cluster_find_and_modify_cmd.cpp +++ b/src/mongo/s/commands/cluster_find_and_modify_cmd.cpp @@ -422,6 +422,8 @@ Status FindAndModifyCmd::explain(OperationContext* opCtx, auto findAndModifyRequest = write_ops::FindAndModifyCommandRequest::parse( IDLParserContext("ClusterFindAndModify"), request.body); if (shouldDoFLERewrite(findAndModifyRequest)) { + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + auto newRequest = processFLEFindAndModifyExplainMongos(opCtx, findAndModifyRequest); return newRequest.first.toBSON(request.body); } else { diff --git a/src/mongo/s/commands/cluster_find_cmd.h b/src/mongo/s/commands/cluster_find_cmd.h index 3b0ff62100c..e8ed843575a 100644 --- a/src/mongo/s/commands/cluster_find_cmd.h +++ b/src/mongo/s/commands/cluster_find_cmd.h @@ -300,10 +300,14 @@ public: if (shouldDoFLERewrite(findCommand)) { invariant(findCommand->getNamespaceOrUUID().nss()); + + if (!findCommand->getEncryptionInformation()->getCrudProcessed().value_or(false)) { + processFLEFindS( + opCtx, findCommand->getNamespaceOrUUID().nss().get(), findCommand.get()); + _didDoFLERewrite = true; + } + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; - processFLEFindS( - opCtx, findCommand->getNamespaceOrUUID().nss().get(), findCommand.get()); - _didDoFLERewrite = true; } return findCommand; diff --git a/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp b/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp index 65b891e4c4c..9fba98c5d56 100644 --- a/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp +++ b/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp @@ -76,10 +76,16 @@ public: bool adminOnly() const final { return false; } + + std::set<StringData> sensitiveFieldNames() const final { + return {CompactStructuredEncryptionData::kCompactionTokensFieldName}; + } } clusterCompactStructuredEncryptionDataCmd; using Cmd = ClusterCompactStructuredEncryptionDataCmd; Cmd::Reply Cmd::Invocation::typedRun(OperationContext* opCtx) { + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + auto nss = request().getNamespace(); const auto dbInfo = uassertStatusOK(Grid::get(opCtx)->catalogCache()->getDatabase(opCtx, nss.db())); diff --git a/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp b/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp index 8d69a923af5..c0d2acb22c1 100644 --- a/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp +++ b/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp @@ -62,6 +62,10 @@ public: return true; } + std::set<StringData> sensitiveFieldNames() const final { + return {GetQueryableEncryptionCountInfo::kTokensFieldName}; + } + class Invocation final : public InvocationBase { public: using InvocationBase::InvocationBase; @@ -96,6 +100,8 @@ public: ClusterGetQueryableEncryptionCountInfoCmd::Reply ClusterGetQueryableEncryptionCountInfoCmd::Invocation::typedRun(OperationContext* opCtx) { + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; + uassert(741502, "FeatureFlagFLE2ProtocolVersion2 is not enabled", gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)); diff --git a/src/mongo/s/commands/cluster_write_cmd.cpp b/src/mongo/s/commands/cluster_write_cmd.cpp index 50ea1af7bbc..169fcc0ca74 100644 --- a/src/mongo/s/commands/cluster_write_cmd.cpp +++ b/src/mongo/s/commands/cluster_write_cmd.cpp @@ -652,9 +652,6 @@ void ClusterWriteCmd::InvocationBase::explain(OperationContext* opCtx, (_batchedRequest.getBatchType() == BatchedCommandRequest::BatchType_Delete || _batchedRequest.getBatchType() == BatchedCommandRequest::BatchType_Update)) { req = processFLEBatchExplain(opCtx, _batchedRequest); - tassert(6636600, - "encryptionInformation should be stripped from request after rewriting for explain", - !req->hasEncryptionInformation()); } auto nss = req ? req->getNS() : _batchedRequest.getNS(); diff --git a/src/mongo/s/query/cluster_aggregate.cpp b/src/mongo/s/query/cluster_aggregate.cpp index 11959054983..6130a80c1fa 100644 --- a/src/mongo/s/query/cluster_aggregate.cpp +++ b/src/mongo/s/query/cluster_aggregate.cpp @@ -410,12 +410,15 @@ Status ClusterAggregate::runAggregate(OperationContext* opCtx, // If the aggregate command supports encrypted collections, do rewrites of the pipeline to // support querying against encrypted fields. if (shouldDoFLERewrite) { - // After this rewriting, the encryption info does not need to be kept around. - pipeline = processFLEPipelineS(opCtx, - namespaces.executionNss, - request.getEncryptionInformation().value(), - std::move(pipeline)); - request.setEncryptionInformation(boost::none); + if (!request.getEncryptionInformation()->getCrudProcessed().value_or(false)) { + pipeline = processFLEPipelineS(opCtx, + namespaces.executionNss, + request.getEncryptionInformation().value(), + std::move(pipeline)); + request.getEncryptionInformation()->setCrudProcessed(true); + } + + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; } pipeline->optimizePipeline(); diff --git a/src/mongo/s/query/cluster_client_cursor.h b/src/mongo/s/query/cluster_client_cursor.h index 3e05b5bac69..c92dab456a3 100644 --- a/src/mongo/s/query/cluster_client_cursor.h +++ b/src/mongo/s/query/cluster_client_cursor.h @@ -253,6 +253,12 @@ public: _leftoverMaxTimeMicros = leftoverMaxTimeMicros; } + /** + * Returns true if operations with this cursor should be omitted from diagnostic sources such as + * currentOp and the profiler. + */ + virtual bool shouldOmitDiagnosticInformation() const = 0; + protected: // Metrics that are accumulated over the lifetime of the cursor, incremented with each getMore. // Useful for diagnostics like telemetry. diff --git a/src/mongo/s/query/cluster_client_cursor_impl.cpp b/src/mongo/s/query/cluster_client_cursor_impl.cpp index 590a970254d..40e3df899b2 100644 --- a/src/mongo/s/query/cluster_client_cursor_impl.cpp +++ b/src/mongo/s/query/cluster_client_cursor_impl.cpp @@ -74,6 +74,7 @@ ClusterClientCursorImpl::ClusterClientCursorImpl(OperationContext* opCtx, _createdDate(opCtx->getServiceContext()->getPreciseClockSource()->now()), _lastUseDate(_createdDate), _queryHash(CurOp::get(opCtx)->debug().queryHash), + _shouldOmitDiagnosticInformation(CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation), _telemetryStoreKey(CurOp::get(opCtx)->debug().telemetryStoreKey) { dassert(!_params.compareWholeSortKeyOnRouter || SimpleBSONObjComparator::kInstance.evaluate( @@ -91,7 +92,8 @@ ClusterClientCursorImpl::ClusterClientCursorImpl(OperationContext* opCtx, _opCtx(opCtx), _createdDate(opCtx->getServiceContext()->getPreciseClockSource()->now()), _lastUseDate(_createdDate), - _queryHash(CurOp::get(opCtx)->debug().queryHash) { + _queryHash(CurOp::get(opCtx)->debug().queryHash), + _shouldOmitDiagnosticInformation(CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation) { dassert(!_params.compareWholeSortKeyOnRouter || SimpleBSONObjComparator::kInstance.evaluate( _params.sortToApplyOnRouter == AsyncResultsMerger::kWholeSortKeySortPattern)); @@ -276,4 +278,8 @@ std::unique_ptr<RouterExecStage> ClusterClientCursorImpl::buildMergerPlan( return root; } +bool ClusterClientCursorImpl::shouldOmitDiagnosticInformation() const { + return _shouldOmitDiagnosticInformation; +} + } // namespace mongo diff --git a/src/mongo/s/query/cluster_client_cursor_impl.h b/src/mongo/s/query/cluster_client_cursor_impl.h index d08ad9d5dab..4f9d988acbb 100644 --- a/src/mongo/s/query/cluster_client_cursor_impl.h +++ b/src/mongo/s/query/cluster_client_cursor_impl.h @@ -116,6 +116,8 @@ public: boost::optional<uint32_t> getQueryHash() const final; + bool shouldOmitDiagnosticInformation() const final; + public: /** * Constructs a CCC whose result set is generated by a mock execution stage. @@ -174,6 +176,10 @@ private: // Whether ClusterClientCursor::next() was interrupted due to MaxTimeMSExpired. bool _maxTimeMSExpired = false; + // Whether to omit information about the getmore that uses this cursor from currentop and the + // profiler. + bool _shouldOmitDiagnosticInformation = false; + // If boost::none, telemetry should not be collected for this cursor. boost::optional<BSONObj> _telemetryStoreKey; diff --git a/src/mongo/s/query/cluster_client_cursor_mock.cpp b/src/mongo/s/query/cluster_client_cursor_mock.cpp index f772973e1c0..33a51ffa3b6 100644 --- a/src/mongo/s/query/cluster_client_cursor_mock.cpp +++ b/src/mongo/s/query/cluster_client_cursor_mock.cpp @@ -162,4 +162,8 @@ boost::optional<repl::ReadConcernArgs> ClusterClientCursorMock::getReadConcern() return boost::none; } +bool ClusterClientCursorMock::shouldOmitDiagnosticInformation() const { + return false; +} + } // namespace mongo diff --git a/src/mongo/s/query/cluster_client_cursor_mock.h b/src/mongo/s/query/cluster_client_cursor_mock.h index 9ab15dd2b6f..2693901193d 100644 --- a/src/mongo/s/query/cluster_client_cursor_mock.h +++ b/src/mongo/s/query/cluster_client_cursor_mock.h @@ -116,6 +116,8 @@ public: */ void queueError(Status status); + bool shouldOmitDiagnosticInformation() const final; + private: bool _killed = false; std::queue<StatusWith<ClusterQueryResult>> _resultsQueue; diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp index 18b9b9dbacf..e3acda4494f 100644 --- a/src/mongo/s/query/cluster_find.cpp +++ b/src/mongo/s/query/cluster_find.cpp @@ -791,6 +791,8 @@ StatusWith<CursorResponse> ClusterFind::runGetMore(OperationContext* opCtx, { CurOp::get(opCtx)->debug().nShards = pinnedCursor.getValue()->getNumRemotes(); CurOp::get(opCtx)->debug().cursorid = cursorId; + CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = + pinnedCursor.getValue()->shouldOmitDiagnosticInformation(); stdx::lock_guard<Client> lk(*opCtx->getClient()); CurOp::get(opCtx)->setOriginatingCommand_inlock( pinnedCursor.getValue()->getOriginatingCommand()); diff --git a/src/mongo/shell/check_log.js b/src/mongo/shell/check_log.js index 20314c34d1d..5aa715dfe1b 100644 --- a/src/mongo/shell/check_log.js +++ b/src/mongo/shell/check_log.js @@ -122,26 +122,9 @@ checkLog = (function() { return actual === expected; }, context = null) { - const logMessages = getGlobalLog(conn); - if (logMessages === null) { - return false; - } + const messages = getFilteredLogMessages(conn, id, attrsDict, severity, isRelaxed, context); - let count = 0; - for (let logMsg of logMessages) { - let obj; - try { - obj = JSON.parse(logMsg); - } catch (ex) { - print('checkLog.checkContainsOnce: JsonJSON.parse() failed: ' + tojson(ex) + ': ' + - logMsg); - throw ex; - } - - if (_compareLogs(obj, id, severity, context, attrsDict, isRelaxed)) { - count++; - } - } + const count = messages.length; return comparator(count, expectedCount); }; @@ -182,6 +165,36 @@ checkLog = (function() { }; /* + * See checkContainsWithCountJson comment. + */ + const getFilteredLogMessages = function( + conn, id, attrsDict, severity = null, isRelaxed = false, context = null) { + const logMessages = getGlobalLog(conn); + if (logMessages === null) { + return false; + } + + let messages = []; + + for (let logMsg of logMessages) { + let obj; + try { + obj = JSON.parse(logMsg); + } catch (ex) { + print('checkLog.checkContainsOnce: JsonJSON.parse() failed: ' + tojson(ex) + ': ' + + logMsg); + throw ex; + } + + if (_compareLogs(obj, id, severity, context, attrsDict, isRelaxed)) { + messages.push(obj); + } + } + + return messages; + }; + + /* * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until * the provided 'msg' is found in the logs, or it times out. Throws an exception on timeout. */ @@ -448,7 +461,8 @@ checkLog = (function() { containsWithCount: containsWithCount, containsWithAtLeastCount: containsWithAtLeastCount, formatAsLogLine: formatAsLogLine, - formatAsJsonLogLine: formatAsJsonLogLine + formatAsJsonLogLine: formatAsJsonLogLine, + getFilteredLogMessages: getFilteredLogMessages, }; })(); })(); |