diff options
author | Shreyas Kalyan <shreyas.kalyan@mongodb.com> | 2023-04-04 17:48:52 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-04-05 16:02:51 +0000 |
commit | 858c7ae1576bdb897a6c8e8162a180aac36dcda8 (patch) | |
tree | 878842e5ef0d8d22f073062ca9e1ac213a69c049 /src/mongo | |
parent | 74641e01217dfcae9a7d14fd4dcc329af2d489c7 (diff) | |
download | mongo-858c7ae1576bdb897a6c8e8162a180aac36dcda8.tar.gz |
SERVER-73303 Remove Queryable Encryption v1 Code
Diffstat (limited to 'src/mongo')
36 files changed, 197 insertions, 3965 deletions
diff --git a/src/mongo/crypto/encryption_fields.idl b/src/mongo/crypto/encryption_fields.idl index cc89e8e6427..61c647ccacf 100644 --- a/src/mongo/crypto/encryption_fields.idl +++ b/src/mongo/crypto/encryption_fields.idl @@ -61,10 +61,10 @@ feature_flags: version: 6.2 cpp_varname: gFeatureFlagFLE2Range default: true - featureFlagFLE2ProtocolVersion2: + featureFlagFLE2CompactForProtocolV2: description: "Enable support for version 2 of Queryable Encryption wire protocol" version: 7.0 - cpp_varname: gFeatureFlagFLE2ProtocolVersion2 + cpp_varname: gFeatureFlagFLE2CompactForProtocolV2 default: true featureFlagFLE2CleanupCommand: description: "Enable support for the Cleanup Structured Encryption Data command" diff --git a/src/mongo/crypto/fle_crypto.cpp b/src/mongo/crypto/fle_crypto.cpp index f88f34a3779..c98541c5877 100644 --- a/src/mongo/crypto/fle_crypto.cpp +++ b/src/mongo/crypto/fle_crypto.cpp @@ -826,21 +826,6 @@ StatusWith<std::vector<uint8_t>> KeyIdAndValue::decrypt(FLEUserKey userKey, */ class EDCClientPayload { public: - // TODO: SERVER-73303 delete v1 functions when v2 is enabled by default - static FLE2InsertUpdatePayload parseInsertUpdatePayload(ConstDataRange cdr); - - static FLE2InsertUpdatePayload serializeInsertUpdatePayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t contentionFactor); - - - static FLE2InsertUpdatePayload serializeInsertUpdatePayloadForRange(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - FLE2RangeInsertSpec spec, - uint8_t sparsity, - uint64_t contentionFactor); - static FLE2InsertUpdatePayloadV2 parseInsertUpdatePayloadV2(ConstDataRange cdr); static FLE2InsertUpdatePayloadV2 serializeInsertUpdatePayloadV2(FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, @@ -854,70 +839,10 @@ public: uint64_t contentionFactor); }; - -FLE2InsertUpdatePayload EDCClientPayload::parseInsertUpdatePayload(ConstDataRange cdr) { - return parseFromCDR<FLE2InsertUpdatePayload>(cdr); -} - FLE2InsertUpdatePayloadV2 EDCClientPayload::parseInsertUpdatePayloadV2(ConstDataRange cdr) { return parseFromCDR<FLE2InsertUpdatePayloadV2>(cdr); } -FLE2InsertUpdatePayload EDCClientPayload::serializeInsertUpdatePayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t contentionFactor) { - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - EDCDerivedFromDataTokenAndContentionFactorToken edcDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateEDCDerivedFromDataTokenAndContentionFactorToken(edcDatakey, contentionFactor); - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, contentionFactor); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, contentionFactor); - - - FLE2InsertUpdatePayload iupayload; - - iupayload.setEdcDerivedToken(edcDataCounterkey.toCDR()); - iupayload.setEscDerivedToken(escDataCounterkey.toCDR()); - iupayload.setEccDerivedToken(eccDataCounterkey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - - - auto swCipherText = KeyIdAndValue::serialize(userKey, value, crypto::aesMode::ctr); - uassertStatusOK(swCipherText); - iupayload.setValue(swCipherText.getValue()); - iupayload.setType(element.type()); - - iupayload.setIndexKeyId(indexKey.keyId); - - return iupayload; -} - FLE2InsertUpdatePayloadV2 EDCClientPayload::serializeInsertUpdatePayloadV2( FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, @@ -1046,60 +971,6 @@ std::unique_ptr<Edges> getEdges(FLE2RangeInsertSpec spec, int sparsity) { } } -// TODO: SERVER-73303 delete when v2 is enabled by default -std::vector<EdgeTokenSet> getEdgeTokenSet(FLE2RangeInsertSpec spec, - int sparsity, - uint64_t contentionFactor, - const EDCToken& edcToken, - const ESCToken& escToken, - const ECCToken& eccToken, - const ECOCToken& ecocToken) { - const auto edges = getEdges(std::move(spec), sparsity); - const auto edgesList = edges->get(); - - std::vector<EdgeTokenSet> tokens; - - for (const auto& edge : edgesList) { - ConstDataRange cdr(edge.rawData(), edge.size()); - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, cdr); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, cdr); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, cdr); - - EDCDerivedFromDataTokenAndContentionFactorToken edcDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateEDCDerivedFromDataTokenAndContentionFactorToken(edcDatakey, - contentionFactor); - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, - contentionFactor); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, - contentionFactor); - - EdgeTokenSet ets; - - ets.setEdcDerivedToken(edcDataCounterkey.toCDR()); - ets.setEscDerivedToken(escDataCounterkey.toCDR()); - ets.setEccDerivedToken(eccDataCounterkey.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey) - .serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - ets.setEncryptedTokens(swEncryptedTokens.getValue()); - - tokens.push_back(ets); - } - - return tokens; -} - std::vector<EdgeTokenSetV2> getEdgeTokenSet( FLE2RangeInsertSpec spec, int sparsity, @@ -1150,69 +1021,6 @@ std::vector<EdgeTokenSetV2> getEdgeTokenSet( return tokens; } -FLE2InsertUpdatePayload EDCClientPayload::serializeInsertUpdatePayloadForRange( - FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - FLE2RangeInsertSpec spec, - uint8_t sparsity, - uint64_t contentionFactor) { - auto element = spec.getValue().getElement(); - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - EDCDerivedFromDataTokenAndContentionFactorToken edcDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateEDCDerivedFromDataTokenAndContentionFactorToken(edcDatakey, contentionFactor); - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, contentionFactor); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, contentionFactor); - - FLE2InsertUpdatePayload iupayload; - - iupayload.setEdcDerivedToken(edcDataCounterkey.toCDR()); - iupayload.setEscDerivedToken(escDataCounterkey.toCDR()); - iupayload.setEccDerivedToken(eccDataCounterkey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - - auto swCipherText = KeyIdAndValue::serialize(userKey, value, crypto::aesMode::ctr); - uassertStatusOK(swCipherText); - iupayload.setValue(swCipherText.getValue()); - iupayload.setType(element.type()); - - iupayload.setIndexKeyId(indexKey.keyId); - - auto edgeTokenSet = - getEdgeTokenSet(spec, sparsity, contentionFactor, edcToken, escToken, eccToken, ecocToken); - - if (!edgeTokenSet.empty()) { - iupayload.setEdgeTokenSet(edgeTokenSet); - } - - return iupayload; -} FLE2InsertUpdatePayloadV2 EDCClientPayload::serializeInsertUpdatePayloadV2ForRange( FLEIndexKeyAndId indexKey, @@ -1482,46 +1290,22 @@ void convertToFLE2Payload(FLEKeyVault* keyVault, if (ep.getType() == Fle2PlaceholderType::kInsert) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2( - indexKey, userKey, el, contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, - iupayload, - builder); - return; - } - - // TODO: SERVER-73303 delete when v2 is enabled by default - auto iupayload = EDCClientPayload::serializeInsertUpdatePayload( + auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2( indexKey, userKey, el, contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayload, + EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, iupayload, builder); - } else if (ep.getType() == Fle2PlaceholderType::kFind) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto findPayload = FLEClientCrypto::serializeFindPayloadV2( - indexKey, userKey, el, ep.getMaxContentionCounter()); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindEqualityPayloadV2, - findPayload, - builder); - return; - } + } else if (ep.getType() == Fle2PlaceholderType::kFind) { - // TODO: SERVER-73303 delete when v2 is enabled by default - auto findpayload = FLEClientCrypto::serializeFindPayload( + auto findPayload = FLEClientCrypto::serializeFindPayloadV2( indexKey, userKey, el, ep.getMaxContentionCounter()); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindEqualityPayload, - findpayload, + EncryptedBinDataType::kFLE2FindEqualityPayloadV2, + findPayload, builder); + } else { uasserted(6410100, "Unsupported Queryable Encryption placeholder type"); } @@ -1539,83 +1323,42 @@ void convertToFLE2Payload(FLEKeyVault* keyVault, << "' is not a valid type for Queryable Encryption Range", isFLE2RangeIndexedSupportedType(elRange.type())); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2ForRange( - indexKey, - userKey, - rangeInsertSpec, - ep.getSparsity().value(), // Enforced as non-optional in this case in IDL - contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, - iupayload, - builder); - return; - } - - // TODO: SERVER-73303 delete when v2 is enabled by default - auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadForRange( + auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2ForRange( indexKey, userKey, rangeInsertSpec, ep.getSparsity().value(), // Enforced as non-optional in this case in IDL contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayload, + EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, iupayload, builder); + } else if (ep.getType() == Fle2PlaceholderType::kFind) { IDLParserContext ctx("root"); auto rangeFindSpec = FLE2RangeFindSpec::parse(ctx, ep.getValue().getElement().Obj()); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - - auto findPayload = [&]() { - if (rangeFindSpec.getEdgesInfo().has_value()) { - auto edges = getMinCover(rangeFindSpec, ep.getSparsity().value()); - - return FLEClientCrypto::serializeFindRangePayloadV2( - indexKey, - userKey, - std::move(edges), - ep.getMaxContentionCounter(), - rangeFindSpec); - } else { - return FLEClientCrypto::serializeFindRangeStubV2(rangeFindSpec); - } - }(); - - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindRangePayloadV2, - findPayload, - builder); - return; - } - - // TODO: SERVER-73303 delete when v2 is enabled by default - auto findpayload = [&]() { + auto findPayload = [&]() { if (rangeFindSpec.getEdgesInfo().has_value()) { auto edges = getMinCover(rangeFindSpec, ep.getSparsity().value()); - return FLEClientCrypto::serializeFindRangePayload( + return FLEClientCrypto::serializeFindRangePayloadV2( indexKey, userKey, std::move(edges), ep.getMaxContentionCounter(), rangeFindSpec); } else { - return FLEClientCrypto::serializeFindRangeStub(rangeFindSpec); + return FLEClientCrypto::serializeFindRangeStubV2(rangeFindSpec); } }(); toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindRangePayload, - findpayload, + EncryptedBinDataType::kFLE2FindRangePayloadV2, + findPayload, builder); + } else { uasserted(6775303, "Unsupported Queryable Encryption placeholder type"); } @@ -1625,17 +1368,10 @@ void convertToFLE2Payload(FLEKeyVault* keyVault, << "' is not a valid type for Queryable Encryption", isFLE2UnindexedSupportedType(el.type())); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto payload = FLE2UnindexedEncryptedValueV2::serialize(userKey, el); - builder->appendBinData( - fieldNameToSerialize, payload.size(), BinDataType::Encrypt, payload.data()); - return; - } - - auto payload = FLE2UnindexedEncryptedValue::serialize(userKey, el); + auto payload = FLE2UnindexedEncryptedValueV2::serialize(userKey, el); builder->appendBinData( fieldNameToSerialize, payload.size(), BinDataType::Encrypt, payload.data()); + } else { uasserted(6338603, "Only Queryable Encryption style encryption placeholders are supported"); @@ -1656,17 +1392,12 @@ void parseAndVerifyInsertUpdatePayload(std::vector<EDCServerPayloadInfo>* pField EDCServerPayloadInfo payloadInfo; payloadInfo.fieldPathName = fieldPath.toString(); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7291901, - "Encountered a Queryable Encryption insert/update payload type that is no " - "longer supported", - type == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2); - auto iupayload = EDCClientPayload::parseInsertUpdatePayloadV2(subCdr); - payloadInfo.payload = VersionedInsertUpdatePayload(std::move(iupayload)); - } else { - auto iupayload = EDCClientPayload::parseInsertUpdatePayload(subCdr); - payloadInfo.payload = VersionedInsertUpdatePayload(std::move(iupayload)); - } + uassert(7291901, + "Encountered a Queryable Encryption insert/update payload type that is no " + "longer supported", + type == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2); + auto iupayload = EDCClientPayload::parseInsertUpdatePayloadV2(subCdr); + payloadInfo.payload = std::move(iupayload); auto bsonType = static_cast<BSONType>(payloadInfo.payload.getType()); @@ -1710,12 +1441,10 @@ void collectEDCServerInfo(std::vector<EDCServerPayloadInfo>* pFields, return; } else if (encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValue || encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7413901, - "Encountered a Queryable Encryption unindexed encrypted payload type that is " - "no longer supported", - encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2); - } + uassert(7413901, + "Encountered a Queryable Encryption unindexed encrypted payload type that is " + "no longer supported", + encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2); return; } uasserted(6373503, @@ -1741,148 +1470,72 @@ void convertServerPayload(ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); - if (encryptedTypeBinding == EncryptedBinDataType::kFLE2FindEqualityPayload || - encryptedTypeBinding == EncryptedBinDataType::kFLE2FindRangePayload || - encryptedTypeBinding == EncryptedBinDataType::kFLE2FindEqualityPayloadV2 || + if (encryptedTypeBinding == EncryptedBinDataType::kFLE2FindEqualityPayloadV2 || encryptedTypeBinding == EncryptedBinDataType::kFLE2FindRangePayloadV2) { builder->appendBinData(fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data<char>()); return; - } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayload || - encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2) { - - // TODO: SERVER-73303 set to just kFLE2InsertUpdatePayloadV2 once is enabled by default - auto validVersionedTypeBinding = - gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility) - ? EncryptedBinDataType::kFLE2InsertUpdatePayloadV2 - : EncryptedBinDataType::kFLE2InsertUpdatePayload; - uassert(7291907, - "Encountered a Queryable Encryption insert/update payload type that is no longer " - "supported", - encryptedTypeBinding == validVersionedTypeBinding); - + } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2) { uassert(6373505, "Unexpected end of iterator", it.it != it.end); const auto payload = it.it; // TODO - validate field is actually indexed in the schema? if (payload->isRangePayload()) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto& v2Payload = payload->payload.getInsertUpdatePayloadVersion2(); - - FLE2IndexedRangeEncryptedValueV2 sp( - v2Payload, EDCServerCollection::generateTags(*payload), payload->counts); - - uassert(7291908, - str::stream() << "Type '" << typeName(sp.bsonType) - << "' is not a valid type for Queryable Encryption Range", - isFLE2RangeIndexedSupportedType(sp.bsonType)); - - std::vector<ServerDerivedFromDataToken> edgeDerivedTokens; - auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - v2Payload.getServerEncryptionToken()); - for (auto& ets : v2Payload.getEdgeTokenSet().value()) { - edgeDerivedTokens.push_back( - FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - ets.getServerDerivedFromDataToken())); - } + auto& v2Payload = payload->payload; - auto swEncrypted = sp.serialize(serverToken, edgeDerivedTokens); - uassertStatusOK(swEncrypted); - toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2RangeIndexedValueV2, - ConstDataRange(swEncrypted.getValue()), - builder); + FLE2IndexedRangeEncryptedValueV2 sp( + v2Payload, EDCServerCollection::generateTags(*payload), payload->counts); - for (auto& mblock : sp.metadataBlocks) { - pTags->push_back({mblock.tag}); - } - it.it++; - return; - } - // TODO: SERVER-73303 delete below once is enabled by default - auto& v1Payload = payload->payload.getInsertUpdatePayloadVersion1(); - FLE2IndexedRangeEncryptedValue sp(v1Payload, payload->counts); - - uassert(6775311, + uassert(7291908, str::stream() << "Type '" << typeName(sp.bsonType) << "' is not a valid type for Queryable Encryption Range", isFLE2RangeIndexedSupportedType(sp.bsonType)); - auto swEncrypted = - sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload->payload.getServerEncryptionToken())); + std::vector<ServerDerivedFromDataToken> edgeDerivedTokens; + auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( + v2Payload.getServerEncryptionToken()); + for (auto& ets : v2Payload.getEdgeTokenSet().value()) { + edgeDerivedTokens.push_back( + FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + ets.getServerDerivedFromDataToken())); + } + + auto swEncrypted = sp.serialize(serverToken, edgeDerivedTokens); uassertStatusOK(swEncrypted); toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2RangeIndexedValue, + EncryptedBinDataType::kFLE2RangeIndexedValueV2, ConstDataRange(swEncrypted.getValue()), builder); - auto tagsRange = EDCServerCollection::generateTags(sp); - for (const auto& tag : tagsRange) { - pTags->push_back({tag}); + for (auto& mblock : sp.metadataBlocks) { + pTags->push_back({mblock.tag}); } } else { dassert(payload->counts.size() == 1); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto tag = EDCServerCollection::generateTag(*payload); - auto& v2Payload = payload->payload.getInsertUpdatePayloadVersion2(); - FLE2IndexedEqualityEncryptedValueV2 sp(v2Payload, tag, payload->counts[0]); - - uassert(7291906, - str::stream() << "Type '" << typeName(sp.bsonType) - << "' is not a valid type for Queryable Encryption Equality", - isFLE2EqualityIndexedSupportedType(sp.bsonType)); - - auto swEncrypted = - sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - v2Payload.getServerEncryptionToken()), - FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - v2Payload.getServerDerivedFromDataToken())); - uassertStatusOK(swEncrypted); - toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2EqualityIndexedValueV2, - ConstDataRange(swEncrypted.getValue()), - builder); - pTags->push_back({tag}); - - it.it++; - return; - } - - // TODO: SERVER-73303 delete below once v2 is enabled by default - auto& v1Payload = payload->payload.getInsertUpdatePayloadVersion1(); - FLE2IndexedEqualityEncryptedValue sp(v1Payload, payload->counts[0]); + auto tag = EDCServerCollection::generateTag(*payload); + auto& v2Payload = payload->payload; + FLE2IndexedEqualityEncryptedValueV2 sp(v2Payload, tag, payload->counts[0]); - uassert(6373506, + uassert(7291906, str::stream() << "Type '" << typeName(sp.bsonType) << "' is not a valid type for Queryable Encryption Equality", isFLE2EqualityIndexedSupportedType(sp.bsonType)); auto swEncrypted = sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload->payload.getServerEncryptionToken())); + v2Payload.getServerEncryptionToken()), + FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + v2Payload.getServerDerivedFromDataToken())); uassertStatusOK(swEncrypted); toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2EqualityIndexedValue, + EncryptedBinDataType::kFLE2EqualityIndexedValueV2, ConstDataRange(swEncrypted.getValue()), builder); - - pTags->push_back({EDCServerCollection::generateTag(*payload)}); + pTags->push_back({tag}); } - } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValue || - encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { - auto validVersionedTypeBinding = - gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility) - ? EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2 - : EncryptedBinDataType::kFLE2UnindexedEncryptedValue; - uassert(7413902, - "Encountered a Queryable Encryption unindexed encrypted payload type that is no " - "longer supported", - encryptedTypeBinding == validVersionedTypeBinding); + } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { builder->appendBinData(fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data()); return; @@ -1927,49 +1580,10 @@ void collectIndexedFields(std::vector<EDCIndexedFields>* pFields, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); - switch (encryptedTypeBinding) { - case EncryptedBinDataType::kFLE2EqualityIndexedValue: - case EncryptedBinDataType::kFLE2RangeIndexedValue: - break; - case EncryptedBinDataType::kFLE2EqualityIndexedValueV2: - case EncryptedBinDataType::kFLE2RangeIndexedValueV2: - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - return; - } - break; - default: - return; - } - pFields->push_back({cdr, fieldPath.toString()}); -} - -void serializeDeletePayload(UUID keyId, FLEKeyVault* keyVault, BSONObjBuilder* builder) { - - auto indexKey = keyVault->getIndexKeyById(keyId); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverEncryptionToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - FLE2DeletePayload payload; - payload.setEcocToken(ecocToken.toCDR()); - payload.setServerEncryptionToken(serverEncryptionToken.toCDR()); - payload.serialize(builder); -} - -BSONObj getDeleteTokensBSON(const EncryptedFieldConfig& ef, FLEKeyVault* keyVault) { - BSONObjBuilder builder; - for (const auto& field : ef.getFields()) { - if (field.getQueries().has_value()) { - BSONObjBuilder subBuilder(builder.subobjStart(field.getPath())); - serializeDeletePayload(field.getKeyId(), keyVault, &subBuilder); - } + if (encryptedTypeBinding == EncryptedBinDataType::kFLE2EqualityIndexedValueV2 || + encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValueV2) { + pFields->push_back({cdr, fieldPath.toString()}); } - - return builder.obj(); } void collectFieldValidationInfo(stdx::unordered_map<std::string, ConstDataRange>* pFields, @@ -2009,19 +1623,6 @@ size_t getEstimatedTagCount(const std::vector<EDCServerPayloadInfo>& serverPaylo return total; } -// TODO: SERVER-73303 remove when v2 is enabled by default -template <typename T> -T decryptAndParseIndexedValue(ConstDataRange cdr, FLEKeyVault* keyVault) { - auto indexKeyId = uassertStatusOK(FLE2IndexedRangeEncryptedValue::readKeyId(cdr)); - - auto indexKey = keyVault->getIndexKeyById(indexKeyId); - - auto serverDataToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - - return uassertStatusOK(T::decryptAndParse(serverDataToken, cdr)); -} - /** * Return the first bit set in a integer. 1 indexed. */ @@ -2593,79 +2194,74 @@ BSONObj FLEClientCrypto::generateCompactionTokens(const EncryptedFieldConfig& cf } BSONObj FLEClientCrypto::decryptDocument(BSONObj& doc, FLEKeyVault* keyVault) { - // TODO: SERVER-73851 remove once libmongocrypt supports parsing v2 payloads - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - BSONObjBuilder builder; + // TODO: SERVER-73851 replace with commented code once mongocrypt supports v2 decryption + // auto crypt = createMongoCrypt(); - auto obj = transformBSON( - doc, [keyVault](ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { - auto [encryptedType, subCdr] = fromEncryptedConstDataRange(cdr); - if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2 || - encryptedType == EncryptedBinDataType::kFLE2RangeIndexedValueV2) { - std::vector<uint8_t> userCipherText; - BSONType type; - if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2) { - auto indexKeyId = - uassertStatusOK(FLE2IndexedEqualityEncryptedValueV2::readKeyId(subCdr)); - auto indexKey = keyVault->getIndexKeyById(indexKeyId); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( - indexKey.key); - userCipherText = uassertStatusOK( - FLE2IndexedEqualityEncryptedValueV2::parseAndDecryptCiphertext( - serverToken, subCdr)); - type = uassertStatusOK( - FLE2IndexedEqualityEncryptedValueV2::readBsonType(subCdr)); - } else { - auto indexKeyId = - uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readKeyId(subCdr)); - auto indexKey = keyVault->getIndexKeyById(indexKeyId); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( - indexKey.key); - userCipherText = uassertStatusOK( - FLE2IndexedRangeEncryptedValueV2::parseAndDecryptCiphertext(serverToken, - subCdr)); - type = - uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readBsonType(subCdr)); - } + // SymmetricKey& key = keyVault->getKMSLocalKey(); + // auto binary = MongoCryptBinary::createFromCDR(ConstDataRange(key.getKey(), + // key.getKeySize())); uassert(7132217, + // "mongocrypt_setopt_kms_provider_local failed", + // mongocrypt_setopt_kms_provider_local(crypt.get(), binary)); - auto userKeyId = uassertStatusOK(KeyIdAndValue::readKeyId(userCipherText)); - auto userKey = keyVault->getUserKeyById(userKeyId); - auto userData = - uassertStatusOK(KeyIdAndValue::decrypt(userKey.key, userCipherText)); - BSONObj obj = toBSON(type, userData); - builder->appendAs(obj.firstElement(), fieldPath); - } else if (encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { - auto [type, userData] = - FLE2UnindexedEncryptedValueV2::deserialize(keyVault, cdr); - BSONObj obj = toBSON(type, userData); - builder->appendAs(obj.firstElement(), fieldPath); - } else { - builder->appendBinData( - fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data()); - } - }); - builder.appendElements(obj); - return builder.obj(); - } + // uassert(7132218, "mongocrypt_init failed", mongocrypt_init(crypt.get())); - auto crypt = createMongoCrypt(); + // UniqueMongoCryptCtx ctx(mongocrypt_ctx_new(crypt.get())); + // auto input = MongoCryptBinary::createFromBSONObj(doc); + // mongocrypt_ctx_decrypt_init(ctx.get(), input); + // BSONObj obj = runStateMachineForDecryption(ctx.get(), keyVault); - SymmetricKey& key = keyVault->getKMSLocalKey(); - auto binary = MongoCryptBinary::createFromCDR(ConstDataRange(key.getKey(), key.getKeySize())); - uassert(7132217, - "mongocrypt_setopt_kms_provider_local failed", - mongocrypt_setopt_kms_provider_local(crypt.get(), binary)); + // return obj; - uassert(7132218, "mongocrypt_init failed", mongocrypt_init(crypt.get())); + BSONObjBuilder builder; - UniqueMongoCryptCtx ctx(mongocrypt_ctx_new(crypt.get())); - auto input = MongoCryptBinary::createFromBSONObj(doc); - mongocrypt_ctx_decrypt_init(ctx.get(), input); - BSONObj obj = runStateMachineForDecryption(ctx.get(), keyVault); + auto obj = transformBSON( + doc, [keyVault](ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { + auto [encryptedType, subCdr] = fromEncryptedConstDataRange(cdr); + if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2 || + encryptedType == EncryptedBinDataType::kFLE2RangeIndexedValueV2) { + std::vector<uint8_t> userCipherText; + BSONType type; + if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2) { + auto indexKeyId = + uassertStatusOK(FLE2IndexedEqualityEncryptedValueV2::readKeyId(subCdr)); + auto indexKey = keyVault->getIndexKeyById(indexKeyId); + auto serverToken = + FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( + indexKey.key); + userCipherText = uassertStatusOK( + FLE2IndexedEqualityEncryptedValueV2::parseAndDecryptCiphertext(serverToken, + subCdr)); + type = + uassertStatusOK(FLE2IndexedEqualityEncryptedValueV2::readBsonType(subCdr)); + } else { + auto indexKeyId = + uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readKeyId(subCdr)); + auto indexKey = keyVault->getIndexKeyById(indexKeyId); + auto serverToken = + FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( + indexKey.key); + userCipherText = + uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::parseAndDecryptCiphertext( + serverToken, subCdr)); + type = uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readBsonType(subCdr)); + } - return obj; + auto userKeyId = uassertStatusOK(KeyIdAndValue::readKeyId(userCipherText)); + auto userKey = keyVault->getUserKeyById(userKeyId); + auto userData = + uassertStatusOK(KeyIdAndValue::decrypt(userKey.key, userCipherText)); + BSONObj obj = toBSON(type, userData); + builder->appendAs(obj.firstElement(), fieldPath); + } else if (encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { + auto [type, userData] = FLE2UnindexedEncryptedValueV2::deserialize(keyVault, cdr); + BSONObj obj = toBSON(type, userData); + builder->appendAs(obj.firstElement(), fieldPath); + } else { + builder->appendBinData(fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data()); + } + }); + builder.appendElements(obj); + return builder.obj(); } void FLEClientCrypto::validateTagsArray(const BSONObj& doc) { @@ -2703,42 +2299,6 @@ void FLEClientCrypto::validateDocument(const BSONObj& doc, auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(field.second); - // TODO: SERVER-73303 delete when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - if (configField->second.getQueries().has_value()) { - if (hasQueryType(configField->second, QueryTypeEnum::Equality)) { - uassert(6371509, - str::stream() << "Field '" << field.first - << "' is marked as equality but not indexed", - encryptedTypeBinding == - EncryptedBinDataType::kFLE2EqualityIndexedValue); - - auto ieev = decryptAndParseIndexedValue<FLE2IndexedEqualityEncryptedValue>( - subCdr, keyVault); - auto tag = EDCServerCollection::generateTag(ieev); - tags.insert({tag, field.first}); - } else if (hasQueryType(configField->second, QueryTypeEnum::RangePreview)) { - uassert(6775316, - str::stream() << "Field '" << field.first - << "' is marked as range but not indexed", - encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValue); - - auto irev = decryptAndParseIndexedValue<FLE2IndexedRangeEncryptedValue>( - subCdr, keyVault); - - auto tagsRange = EDCServerCollection::generateTags(irev); - for (const auto& tag : tagsRange) { - tags.insert({tag, field.first}); - } - } - } else { - uassert(6379105, - str::stream() << "Field '" << field.first << "' must be marked unindexed", - encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValue); - } - continue; - } - if (configField->second.getQueries().has_value()) { if (hasQueryType(configField->second, QueryTypeEnum::Equality)) { uassert(7293205, @@ -3264,38 +2824,6 @@ StatusWith<ECCNullDocument> ECCCollection::decryptNullDocument(ECCTwiceDerivedVa } -FLE2FindEqualityPayload FLEClientCrypto::serializeFindPayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t maxContentionFactor) { - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - FLE2FindEqualityPayload payload; - - payload.setEdcDerivedToken(edcDatakey.toCDR()); - payload.setEscDerivedToken(escDatakey.toCDR()); - payload.setEccDerivedToken(eccDatakey.toCDR()); - payload.setMaxCounter(maxContentionFactor); - payload.setServerEncryptionToken(serverToken.toCDR()); - - return payload; -} - FLE2FindEqualityPayloadV2 FLEClientCrypto::serializeFindPayloadV2(FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, BSONElement element, @@ -3326,54 +2854,6 @@ FLE2FindEqualityPayloadV2 FLEClientCrypto::serializeFindPayloadV2(FLEIndexKeyAnd return payload; } -FLE2FindRangePayload FLEClientCrypto::serializeFindRangePayload( - FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - const std::vector<std::string>& edges, - uint64_t maxContentionFactor, - const FLE2RangeFindSpec& spec) { - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - - // TODO - should we randomize the sort order of the edges vector? - std::vector<EdgeFindTokenSet> tokens; - for (auto const& edge : edges) { - - ConstDataRange value(edge.c_str(), edge.size()); - - EdgeFindTokenSet tokenSet; - tokenSet.setEdcDerivedToken( - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value) - .toCDR()); - - tokenSet.setEscDerivedToken( - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value) - .toCDR()); - tokenSet.setEccDerivedToken( - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value) - .toCDR()); - tokens.push_back(std::move(tokenSet)); - } - - FLE2FindRangePayload payload; - FLE2FindRangePayloadEdgesInfo edgesInfo; - - edgesInfo.setEdges(std::move(tokens)); - edgesInfo.setMaxCounter(maxContentionFactor); - edgesInfo.setServerEncryptionToken(serverToken.toCDR()); - - payload.setPayload(edgesInfo); - payload.setFirstOperator(spec.getFirstOperator()); - payload.setSecondOperator(spec.getSecondOperator()); - payload.setPayloadId(spec.getPayloadId()); - - return payload; -} FLE2FindRangePayloadV2 FLEClientCrypto::serializeFindRangePayloadV2( FLEIndexKeyAndId indexKey, @@ -3421,16 +2901,6 @@ FLE2FindRangePayloadV2 FLEClientCrypto::serializeFindRangePayloadV2( return payload; } -FLE2FindRangePayload FLEClientCrypto::serializeFindRangeStub(const FLE2RangeFindSpec& spec) { - FLE2FindRangePayload payload; - - payload.setFirstOperator(spec.getFirstOperator()); - payload.setSecondOperator(spec.getSecondOperator()); - payload.setPayloadId(spec.getPayloadId()); - - return payload; -} - FLE2FindRangePayloadV2 FLEClientCrypto::serializeFindRangeStubV2(const FLE2RangeFindSpec& spec) { FLE2FindRangePayloadV2 payload; @@ -3479,21 +2949,6 @@ BSONObj ECOCCollection::generateDocument(StringData fieldName, ConstDataRange pa return builder.obj(); } -ECOCCompactionDocument ECOCCollection::parseAndDecrypt(const BSONObj& doc, ECOCToken token) { - IDLParserContext ctx("root"); - auto ecocDoc = EcocDocument::parse(ctx, doc); - - auto swTokens = EncryptedStateCollectionTokens::decryptAndParse(token, ecocDoc.getValue()); - uassertStatusOK(swTokens); - auto& keys = swTokens.getValue(); - - ECOCCompactionDocument ret; - ret.fieldName = ecocDoc.getFieldName().toString(); - ret.esc = keys.esc; - ret.ecc = keys.ecc; - return ret; -} - ECOCCompactionDocumentV2 ECOCCollection::parseAndDecryptV2(const BSONObj& doc, ECOCToken token) { IDLParserContext ctx("root"); auto ecocDoc = EcocDocument::parse(ctx, doc); @@ -3508,173 +2963,6 @@ ECOCCompactionDocumentV2 ECOCCollection::parseAndDecryptV2(const BSONObj& doc, E return ret; } -FLE2IndexedEqualityEncryptedValue::FLE2IndexedEqualityEncryptedValue( - FLE2InsertUpdatePayload payload, uint64_t counter) - : edc(FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEdcDerivedToken())), - esc(FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEscDerivedToken())), - ecc(FLETokenFromCDR<FLETokenType::ECCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEccDerivedToken())), - count(counter), - bsonType(static_cast<BSONType>(payload.getType())), - indexKeyId(payload.getIndexKeyId()), - clientEncryptedValue(FLEUtil::vectorFromCDR(payload.getValue())) { - uassert(6373508, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload", - isValidBSONType(payload.getType())); -} - -FLE2IndexedEqualityEncryptedValue::FLE2IndexedEqualityEncryptedValue( - EDCDerivedFromDataTokenAndContentionFactorToken edcParam, - ESCDerivedFromDataTokenAndContentionFactorToken escParam, - ECCDerivedFromDataTokenAndContentionFactorToken eccParam, - uint64_t countParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> clientEncryptedValueParam) - : edc(edcParam), - esc(escParam), - ecc(eccParam), - count(countParam), - bsonType(typeParam), - indexKeyId(indexKeyIdParam), - clientEncryptedValue(clientEncryptedValueParam) {} - -StatusWith<UUID> FLE2IndexedEqualityEncryptedValue::readKeyId( - ConstDataRange serializedServerValue) { - ConstDataRangeCursor baseCdrc(serializedServerValue); - - auto swKeyId = baseCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swKeyId.isOK()) { - return {swKeyId.getStatus()}; - } - - return UUID::fromCDR(swKeyId.getValue()); -} - -StatusWith<FLE2IndexedEqualityEncryptedValue> FLE2IndexedEqualityEncryptedValue::decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { - ConstDataRangeCursor serializedServerCdrc(serializedServerValue); - - auto swIndexKeyId = serializedServerCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swIndexKeyId.isOK()) { - return {swIndexKeyId.getStatus()}; - } - - UUID indexKey = UUID::fromCDR(swIndexKeyId.getValue()); - - auto swBsonType = serializedServerCdrc.readAndAdvanceNoThrow<uint8_t>(); - if (!swBsonType.isOK()) { - return {swBsonType.getStatus()}; - } - - uassert(6373509, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload", - isValidBSONType(swBsonType.getValue())); - - auto type = static_cast<BSONType>(swBsonType.getValue()); - - auto swVec = FLEUtil::decryptData(token.toCDR(), serializedServerCdrc); - if (!swVec.isOK()) { - return swVec.getStatus(); - } - - auto data = swVec.getValue(); - - ConstDataRangeCursor serverEncryptedValueCdrc(data); - - auto swLength = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swLength.isOK()) { - return {swLength.getStatus()}; - } - - std::uint64_t length = swLength.getValue(); - - auto start = serverEncryptedValueCdrc.data(); - - auto advance = serverEncryptedValueCdrc.advanceNoThrow(length); - if (!advance.isOK()) { - return advance; - } - - std::vector<uint8_t> cipherText(length); - - std::copy(start, start + length, cipherText.data()); - - auto swCount = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swCount.isOK()) { - return {swCount.getStatus()}; - } - - auto swEdc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEdc.isOK()) { - return swEdc.getStatus(); - } - - auto swEsc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEsc.isOK()) { - return swEsc.getStatus(); - } - - auto swEcc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEcc.isOK()) { - return swEcc.getStatus(); - } - - return FLE2IndexedEqualityEncryptedValue( - EDCDerivedFromDataTokenAndContentionFactorToken(swEdc.getValue()), - ESCDerivedFromDataTokenAndContentionFactorToken(swEsc.getValue()), - ECCDerivedFromDataTokenAndContentionFactorToken(swEcc.getValue()), - swCount.getValue(), - type, - indexKey, - std::move(cipherText)); -} - - -StatusWith<std::vector<uint8_t>> FLE2IndexedEqualityEncryptedValue::serialize( - ServerDataEncryptionLevel1Token token) { - BufBuilder builder(clientEncryptedValue.size() + sizeof(uint64_t) * 2 + sizeof(PrfBlock) * 3); - - - builder.appendNum(LittleEndian<uint64_t>(clientEncryptedValue.size())); - - builder.appendBuf(clientEncryptedValue.data(), clientEncryptedValue.size()); - - builder.appendNum(LittleEndian<uint64_t>(count)); - - builder.appendStruct(edc.data); - - builder.appendStruct(esc.data); - - builder.appendStruct(ecc.data); - - dassert(builder.len() == - static_cast<int>(clientEncryptedValue.size() + sizeof(uint64_t) * 2 + - sizeof(PrfBlock) * 3)); - - auto swEncryptedData = encryptData(token.toCDR(), ConstDataRange(builder.buf(), builder.len())); - if (!swEncryptedData.isOK()) { - return swEncryptedData; - } - - auto cdrKeyId = indexKeyId.toCDR(); - auto serverEncryptedValue = swEncryptedData.getValue(); - - std::vector<uint8_t> serializedServerValue(serverEncryptedValue.size() + cdrKeyId.length() + 1); - - std::copy(cdrKeyId.data(), cdrKeyId.data() + cdrKeyId.length(), serializedServerValue.begin()); - uint8_t bsonTypeByte = bsonType; - std::copy( - &bsonTypeByte, (&bsonTypeByte) + 1, serializedServerValue.begin() + cdrKeyId.length()); - std::copy(serverEncryptedValue.begin(), - serverEncryptedValue.end(), - serializedServerValue.begin() + cdrKeyId.length() + 1); - - return serializedServerValue; -} - FLE2TagAndEncryptedMetadataBlock::FLE2TagAndEncryptedMetadataBlock(uint64_t countParam, uint64_t contentionParam, PrfBlock tagParam) @@ -3983,11 +3271,6 @@ std::vector<uint8_t> serializeUnindexedEncryptedValue(const FLEUserKeyAndId& use return buf; } -std::vector<uint8_t> FLE2UnindexedEncryptedValue::serialize(const FLEUserKeyAndId& userKey, - const BSONElement& element) { - return serializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValue>(userKey, element); -} - std::vector<uint8_t> FLE2UnindexedEncryptedValueV2::serialize(const FLEUserKeyAndId& userKey, const BSONElement& element) { return serializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValueV2>(userKey, element); @@ -4015,331 +3298,11 @@ std::pair<BSONType, std::vector<uint8_t>> deserializeUnindexedEncryptedValue(FLE return {bsonType, data}; } -std::pair<BSONType, std::vector<uint8_t>> FLE2UnindexedEncryptedValue::deserialize( - FLEKeyVault* keyVault, ConstDataRange blob) { - return deserializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValue>(keyVault, blob); -} - std::pair<BSONType, std::vector<uint8_t>> FLE2UnindexedEncryptedValueV2::deserialize( FLEKeyVault* keyVault, ConstDataRange blob) { return deserializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValueV2>(keyVault, blob); } -std::vector<FLEEdgeToken> toFLEEdgeTokenSet(const FLE2InsertUpdatePayload& payload) { - const auto ets = payload.getEdgeTokenSet().get(); - std::vector<FLEEdgeToken> tokens; - tokens.reserve(ets.size()); - - for (const auto& et : ets) { - FLEEdgeToken edgeToken; - edgeToken.edc = - FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( - et.getEdcDerivedToken()); - edgeToken.esc = - FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( - et.getEscDerivedToken()); - edgeToken.ecc = - FLETokenFromCDR<FLETokenType::ECCDerivedFromDataTokenAndContentionFactorToken>( - et.getEccDerivedToken()); - - tokens.push_back(edgeToken); - } - - return tokens; -} - -FLE2IndexedRangeEncryptedValue::FLE2IndexedRangeEncryptedValue(FLE2InsertUpdatePayload payload, - std::vector<uint64_t> countersParam) - : tokens(toFLEEdgeTokenSet(payload)), - counters(std::move(countersParam)), - bsonType(static_cast<BSONType>(payload.getType())), - indexKeyId(payload.getIndexKeyId()), - clientEncryptedValue(FLEUtil::vectorFromCDR(payload.getValue())) { - uassert(6775312, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload", - isValidBSONType(payload.getType())); - uassert( - 6775313, "Mismatch between tokens and counters count", tokens.size() == counters.size()); -} - -FLE2IndexedRangeEncryptedValue::FLE2IndexedRangeEncryptedValue( - std::vector<FLEEdgeToken> tokenSet, - std::vector<uint64_t> countersParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> clientEncryptedValueParam) - : tokens(std::move(tokenSet)), - counters(countersParam), - bsonType(typeParam), - indexKeyId(indexKeyIdParam), - clientEncryptedValue(clientEncryptedValueParam) { - uassert( - 6775314, "Mismatch between tokens and counters count", tokens.size() == counters.size()); -} - -StatusWith<UUID> FLE2IndexedRangeEncryptedValue::readKeyId(ConstDataRange serializedServerValue) { - ConstDataRangeCursor baseCdrc(serializedServerValue); - - auto swKeyId = baseCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swKeyId.isOK()) { - return {swKeyId.getStatus()}; - } - - return UUID::fromCDR(swKeyId.getValue()); -} - -StatusWith<FLE2IndexedRangeEncryptedValue> FLE2IndexedRangeEncryptedValue::decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { - ConstDataRangeCursor serializedServerCdrc(serializedServerValue); - - auto swIndexKeyId = serializedServerCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swIndexKeyId.isOK()) { - return {swIndexKeyId.getStatus()}; - } - - UUID indexKey = UUID::fromCDR(swIndexKeyId.getValue()); - - auto swBsonType = serializedServerCdrc.readAndAdvanceNoThrow<uint8_t>(); - if (!swBsonType.isOK()) { - return {swBsonType.getStatus()}; - } - - uassert(6775310, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload Range", - isValidBSONType(swBsonType.getValue())); - - auto type = static_cast<BSONType>(swBsonType.getValue()); - - auto swVec = FLEUtil::decryptData(token.toCDR(), serializedServerCdrc); - if (!swVec.isOK()) { - return swVec.getStatus(); - } - - auto data = swVec.getValue(); - - ConstDataRangeCursor serverEncryptedValueCdrc(data); - - auto swLength = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swLength.isOK()) { - return {swLength.getStatus()}; - } - - std::uint64_t length = swLength.getValue(); - - auto start = serverEncryptedValueCdrc.data(); - - auto advance = serverEncryptedValueCdrc.advanceNoThrow(length); - if (!advance.isOK()) { - return advance; - } - - std::vector<uint8_t> cipherText(length); - - std::copy(start, start + length, cipherText.data()); - - auto swEdgeCount = - serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint32_t>>(); - if (!swEdgeCount.isOK()) { - return {swEdgeCount.getStatus()}; - } - - uassert(6775315, "Edge count must not be over 129", swEdgeCount.getValue() <= 129); - - std::vector<FLEEdgeToken> tokens; - tokens.reserve(swEdgeCount.getValue()); - - std::vector<uint64_t> counters; - counters.reserve(swEdgeCount.getValue()); - - for (size_t i = 0; i < swEdgeCount.getValue(); i++) { - - auto swEdc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEdc.isOK()) { - return swEdc.getStatus(); - } - - auto swEsc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEsc.isOK()) { - return swEsc.getStatus(); - } - - auto swEcc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEcc.isOK()) { - return swEcc.getStatus(); - } - tokens.push_back( - FLEEdgeToken{EDCDerivedFromDataTokenAndContentionFactorToken(swEdc.getValue()), - ESCDerivedFromDataTokenAndContentionFactorToken(swEsc.getValue()), - ECCDerivedFromDataTokenAndContentionFactorToken(swEcc.getValue())}); - - auto swCount = - serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swCount.isOK()) { - return {swCount.getStatus()}; - } - counters.push_back(swCount.getValue()); - } - - return FLE2IndexedRangeEncryptedValue(tokens, counters, type, indexKey, std::move(cipherText)); -} - - -StatusWith<std::vector<uint8_t>> FLE2IndexedRangeEncryptedValue::serialize( - ServerDataEncryptionLevel1Token token) { - BufBuilder builder(clientEncryptedValue.size() + sizeof(uint64_t) * 1 + - sizeof(uint64_t) * counters.size() + sizeof(uint32_t) + - tokens.size() * sizeof(PrfBlock) * 3); - - - builder.appendNum(LittleEndian<uint64_t>(clientEncryptedValue.size())); - - builder.appendBuf(clientEncryptedValue.data(), clientEncryptedValue.size()); - - builder.appendNum(LittleEndian<uint32_t>(tokens.size())); - - size_t c = 0; - for (auto const& ets : tokens) { - - builder.appendStruct(ets.edc.data); - - builder.appendStruct(ets.esc.data); - - builder.appendStruct(ets.ecc.data); - - builder.appendNum(LittleEndian<uint64_t>(counters[c++])); - } - - dassert(builder.len() == - static_cast<int>(clientEncryptedValue.size() + sizeof(uint64_t) * 1 + - sizeof(uint64_t) * counters.size() + sizeof(uint32_t) + - sizeof(PrfBlock) * 3 * tokens.size())); - - auto swEncryptedData = encryptData(token.toCDR(), ConstDataRange(builder.buf(), builder.len())); - if (!swEncryptedData.isOK()) { - return swEncryptedData; - } - - auto cdrKeyId = indexKeyId.toCDR(); - auto serverEncryptedValue = swEncryptedData.getValue(); - - std::vector<uint8_t> serializedServerValue(serverEncryptedValue.size() + cdrKeyId.length() + 1); - - std::copy(cdrKeyId.data(), cdrKeyId.data() + cdrKeyId.length(), serializedServerValue.begin()); - uint8_t bsonTypeByte = bsonType; - std::copy( - &bsonTypeByte, (&bsonTypeByte) + 1, serializedServerValue.begin() + cdrKeyId.length()); - std::copy(serverEncryptedValue.begin(), - serverEncryptedValue.end(), - serializedServerValue.begin() + cdrKeyId.length() + 1); - - return serializedServerValue; -} - -VersionedEdgeTokenSet::VersionedEdgeTokenSet(EdgeTokenSet ets) : edgeTokenSet(std::move(ets)) {} - -VersionedEdgeTokenSet::VersionedEdgeTokenSet(EdgeTokenSetV2 ets) : edgeTokenSet(std::move(ets)) {} - -ConstDataRange VersionedEdgeTokenSet::getEscDerivedToken() const { - return stdx::visit( - OverloadedVisitor{[](const EdgeTokenSet& ets) { return ets.getEscDerivedToken(); }, - [](const EdgeTokenSetV2& ets) { - return ets.getEscDerivedToken(); - }}, - edgeTokenSet); -} - -ConstDataRange VersionedEdgeTokenSet::getEncryptedTokens() const { - return stdx::visit( - OverloadedVisitor{[](const EdgeTokenSet& ets) { return ets.getEncryptedTokens(); }, - [](const EdgeTokenSetV2& ets) { - return ets.getEncryptedTokens(); - }}, - edgeTokenSet); -} - -VersionedInsertUpdatePayload::VersionedInsertUpdatePayload(FLE2InsertUpdatePayload iup) - : iupayload(std::move(iup)), edgeTokenSet(convertPayloadEdgeTokenSet<decltype(iup)>()) {} - -VersionedInsertUpdatePayload::VersionedInsertUpdatePayload(FLE2InsertUpdatePayloadV2 iup) - : iupayload(std::move(iup)), edgeTokenSet(convertPayloadEdgeTokenSet<decltype(iup)>()) {} - -const FLE2InsertUpdatePayload& VersionedInsertUpdatePayload::getInsertUpdatePayloadVersion1() - const { - auto payloadPtr = stdx::get_if<FLE2InsertUpdatePayload>(&iupayload); - uassert( - 7291904, "Attempted to retrieve invalid version of FLE2InsertUpdatePayload", payloadPtr); - return *payloadPtr; -} - -const FLE2InsertUpdatePayloadV2& VersionedInsertUpdatePayload::getInsertUpdatePayloadVersion2() - const { - auto payloadPtr = stdx::get_if<FLE2InsertUpdatePayloadV2>(&iupayload); - uassert( - 7291905, "Attempted to retrieve invalid version of FLE2InsertUpdatePayload", payloadPtr); - return *payloadPtr; -} - -const mongo::UUID& VersionedInsertUpdatePayload::getIndexKeyId() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) -> const mongo::UUID& { - return v1.getIndexKeyId(); - }, - [](const FLE2InsertUpdatePayloadV2& v2) -> const mongo::UUID& { - return v2.getIndexKeyId(); - }}, - iupayload); -} - -int VersionedInsertUpdatePayload::getType() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getType(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getType(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getEncryptedTokens() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getEncryptedTokens(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getEncryptedTokens(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getEscDerivedToken() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getEscDerivedToken(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getEscDerivedToken(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getEdcDerivedToken() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getEdcDerivedToken(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getEdcDerivedToken(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getServerEncryptionToken() const { - return stdx::visit(OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { - return v1.getServerEncryptionToken(); - }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getServerEncryptionToken(); - }}, - iupayload); -} - -const boost::optional<std::vector<VersionedEdgeTokenSet>>& -VersionedInsertUpdatePayload::getEdgeTokenSet() const { - return edgeTokenSet; -} FLE2IndexedRangeEncryptedValueV2::FLE2IndexedRangeEncryptedValueV2( const FLE2InsertUpdatePayloadV2& payload, @@ -4451,11 +3414,8 @@ FLE2IndexedRangeEncryptedValueV2::parseAndValidateFields(ConstDataRange serializ sizeof(FLE2TagAndEncryptedMetadataBlock::SerializedBlob))); } - return {{UUID::fromCDR(swIndexKeyId.getValue()), - type, - edgeCount, - encryptedDataCdrc, - metadataBlocks}}; + return FLE2IndexedRangeEncryptedValueV2::ParsedFields{ + UUID::fromCDR(swIndexKeyId.getValue()), type, edgeCount, encryptedDataCdrc, metadataBlocks}; } StatusWith<std::vector<uint8_t>> FLE2IndexedRangeEncryptedValueV2::parseAndDecryptCiphertext( @@ -4606,13 +3566,8 @@ void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, visitEncryptedBSON(obj, [&indexedFields](ConstDataRange cdr, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); - auto expectedPayloadType = EncryptedBinDataType::kFLE2InsertUpdatePayload; - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - expectedPayloadType = EncryptedBinDataType::kFLE2InsertUpdatePayloadV2; - } - - if (encryptedTypeBinding == expectedPayloadType) { + if (encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2) { uassert(6373601, str::stream() << "Field '" << fieldPath << "' is encrypted, but absent from schema", @@ -4627,12 +3582,6 @@ void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, } void EDCServerCollection::validateModifiedDocumentCompatibility(BSONObj& obj) { - - // TODO: SERVER-73303 delete when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return; - } - visitEncryptedBSON(obj, [](ConstDataRange cdr, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); switch (encryptedTypeBinding) { @@ -4683,40 +3632,9 @@ PrfBlock EDCServerCollection::generateTag(const EDCServerPayloadInfo& payload) { return generateTag(edcTwiceDerived, payload.counts[0]); } -// TODO: SERVER-73303 delete when v2 is enabled by default -PrfBlock EDCServerCollection::generateTag(const FLE2IndexedEqualityEncryptedValue& indexedValue) { - auto edcTwiceDerived = - FLETwiceDerivedTokenGenerator::generateEDCTwiceDerivedToken(indexedValue.edc); - return generateTag(edcTwiceDerived, indexedValue.count); -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -PrfBlock EDCServerCollection::generateTag(const FLEEdgeToken& token, FLECounter count) { - auto edcTwiceDerived = FLETwiceDerivedTokenGenerator::generateEDCTwiceDerivedToken(token.edc); - return generateTag(edcTwiceDerived, count); -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -std::vector<PrfBlock> EDCServerCollection::generateTags( - const FLE2IndexedRangeEncryptedValue& indexedValue) { - uassert(6775317, - "Mismatch between tokens and counters count", - indexedValue.tokens.size() == indexedValue.counters.size()); - - std::vector<PrfBlock> tags; - tags.reserve(indexedValue.tokens.size()); - - for (size_t i = 0; i < indexedValue.tokens.size(); i++) { - tags.push_back( - EDCServerCollection::generateTag(indexedValue.tokens[i], indexedValue.counters[i])); - } - - return tags; -} - std::vector<PrfBlock> EDCServerCollection::generateTags(const EDCServerPayloadInfo& rangePayload) { // throws if EDCServerPayloadInfo has invalid payload version - auto& v2Payload = rangePayload.payload.getInsertUpdatePayloadVersion2(); + auto& v2Payload = rangePayload.payload; uassert(7291909, "InsertUpdatePayload must have an edge token set", @@ -4738,30 +3656,6 @@ std::vector<PrfBlock> EDCServerCollection::generateTags(const EDCServerPayloadIn return tags; } -StatusWith<FLE2IndexedEqualityEncryptedValue> EDCServerCollection::decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { - auto pair = fromEncryptedConstDataRange(serializedServerValue); - uassert(6672412, - "Wrong encrypted field type", - pair.first == EncryptedBinDataType::kFLE2EqualityIndexedValue); - - return FLE2IndexedEqualityEncryptedValue::decryptAndParse(token, pair.second); -} - -StatusWith<FLE2IndexedEqualityEncryptedValue> EDCServerCollection::decryptAndParse( - ConstDataRange token, ConstDataRange serializedServerValue) { - auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>(token); - - return FLE2IndexedEqualityEncryptedValue::decryptAndParse(serverToken, serializedServerValue); -} - -StatusWith<FLE2IndexedRangeEncryptedValue> EDCServerCollection::decryptAndParseRange( - ConstDataRange token, ConstDataRange serializedServerValue) { - auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>(token); - - return FLE2IndexedRangeEncryptedValue::decryptAndParse(serverToken, serializedServerValue); -} - std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> EDCServerCollection::generateEDCTokens( EDCDerivedFromDataToken token, uint64_t maxContentionFactor) { std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> tokens; @@ -4905,62 +3799,6 @@ BSONObj EDCServerCollection::finalizeForUpdate( return builder.obj(); } -BSONObj EDCServerCollection::generateUpdateToRemoveTags( - const std::vector<EDCIndexedFields>& removedFields, const StringMap<FLEDeleteToken>& tokenMap) { - std::vector<TagInfo> tags; - - for (const auto& field : removedFields) { - auto tokenIt = tokenMap.find(field.fieldPathName); - uassert(6371513, - str::stream() << "Could not find field'" << field.fieldPathName - << "' in delete token map.", - tokenIt != tokenMap.end()); - auto token = *tokenIt; - - auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(field.value); - auto localEncryptedTypeBinding = - encryptedTypeBinding; // Workaround the fact that we cannot pass a structured binding - // variable into a lambda - uassert(6371514, - str::stream() << "Field'" << field.fieldPathName - << "' in not a supported encrypted type: " - << EncryptedBinDataType_serializer(localEncryptedTypeBinding), - encryptedTypeBinding == EncryptedBinDataType::kFLE2EqualityIndexedValue || - encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValue); - - if (encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValue) { - auto range = uassertStatusOK(FLE2IndexedRangeEncryptedValue::decryptAndParse( - token.second.serverEncryptionToken, subCdr)); - - for (size_t i = 0; i < range.counters.size(); i++) { - auto tag = EDCServerCollection::generateTag(range.tokens[i], range.counters[i]); - tags.push_back({tag}); - } - } else { - auto ieev = uassertStatusOK(FLE2IndexedEqualityEncryptedValue::decryptAndParse( - token.second.serverEncryptionToken, subCdr)); - auto tag = EDCServerCollection::generateTag(ieev); - - tags.push_back({tag}); - } - } - - // Build { $pull : {__safeContent__ : {$in : [tag..] } } } - BSONObjBuilder builder; - { - BSONObjBuilder subBuilder(builder.subobjStart(kDollarPull)); - BSONObjBuilder pushBuilder(subBuilder.subobjStart(kSafeContent)); - BSONArrayBuilder arrayBuilder(pushBuilder.subarrayStart(kDollarIn)); - - // Add new tags - for (auto const& tag : tags) { - appendTag(tag.tag, &arrayBuilder); - } - } - - return builder.obj(); -} - BSONObj EDCServerCollection::generateUpdateToRemoveTags(const std::vector<PrfBlock>& tagsToPull) { uassert(7293203, "Cannot generate update command to remove tags with empty tags", @@ -5063,10 +3901,6 @@ EncryptedFieldConfig EncryptionInformationHelpers::getAndValidateSchema( auto efc = EncryptedFieldConfig::parse(IDLParserContext("schema"), element.Obj()); - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(6371206, "Expected a value for eccCollection", efc.getEccCollection().has_value()); - } uassert(6371207, "Expected a value for escCollection", efc.getEscCollection().has_value()); uassert(6371208, "Expected a value for ecocCollection", efc.getEcocCollection().has_value()); @@ -5083,48 +3917,6 @@ std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(Cons return {subType, cdrc}; } -BSONObj EncryptionInformationHelpers::encryptionInformationSerializeForDelete( - const NamespaceString& nss, const EncryptedFieldConfig& ef, FLEKeyVault* keyVault) { - - EncryptionInformation ei; - ei.setType(kEncryptionInformationSchemaVersion); - - ei.setSchema(BSON(nss.toString() << ef.toBSON())); - - ei.setDeleteTokens(BSON(nss.toString() << getDeleteTokensBSON(ef, keyVault))); - - return ei.toBSON(); -} - -StringMap<FLEDeleteToken> EncryptionInformationHelpers::getDeleteTokens( - const NamespaceString& nss, const EncryptionInformation& ei) { - uassert(6371308, "DeleteTokens is empty", ei.getDeleteTokens().has_value()); - - BSONObj deleteTokens = ei.getDeleteTokens().value(); - - auto element = deleteTokens.getField(nss.toString()); - - uassert(6371309, - "DeleteTokens item for namespace is not set", - !element.eoo() && element.type() == Object); - - StringMap<FLEDeleteToken> map; - for (const auto& deleteTokenBSON : element.Obj()) { - uassert(6371310, "DeleteToken is not an object", deleteTokenBSON.type() == Object); - - auto payload = FLE2DeletePayload::parse(IDLParserContext("delete"), deleteTokenBSON.Obj()); - - auto deleteToken = - FLEDeleteToken{FLETokenFromCDR<FLETokenType::ECOCToken>(payload.getEcocToken()), - FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload.getServerEncryptionToken())}; - - map.insert({deleteTokenBSON.fieldNameStringData().toString(), deleteToken}); - } - - return map; -} - ParsedFindEqualityPayload::ParsedFindEqualityPayload(BSONElement fleFindPayload) : ParsedFindEqualityPayload(binDataToCDR(fleFindPayload)){}; @@ -5135,41 +3927,17 @@ ParsedFindEqualityPayload::ParsedFindEqualityPayload(ConstDataRange cdr) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); auto encryptedType = encryptedTypeBinding; - // TODO: SERVER-73303 refactor when v2 is enabled by default - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7292600, - str::stream() << "Unexpected encrypted payload type: " - << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindEqualityPayloadV2); - - auto payload = parseFromCDR<FLE2FindEqualityPayloadV2>(subCdr); - - escToken = - FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(payload.getEscDerivedToken()); - edcToken = - FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(payload.getEdcDerivedToken()); - serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - payload.getServerDerivedFromDataToken()); - maxCounter = payload.getMaxCounter(); - return; - } - - uassert(6435600, + uassert(7292600, str::stream() << "Unexpected encrypted payload type: " << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindEqualityPayload); + encryptedType == EncryptedBinDataType::kFLE2FindEqualityPayloadV2); - auto payload = parseFromCDR<FLE2FindEqualityPayload>(subCdr); + auto payload = parseFromCDR<FLE2FindEqualityPayloadV2>(subCdr); escToken = FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(payload.getEscDerivedToken()); - eccToken = FLETokenFromCDR<FLETokenType::ECCDerivedFromDataToken>(payload.getEccDerivedToken()); edcToken = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(payload.getEdcDerivedToken()); - - if (payload.getServerEncryptionToken().has_value()) { - serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload.getServerEncryptionToken().value()); - } - + serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + payload.getServerDerivedFromDataToken()); maxCounter = payload.getMaxCounter(); } @@ -5183,46 +3951,12 @@ ParsedFindRangePayload::ParsedFindRangePayload(ConstDataRange cdr) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); auto encryptedType = encryptedTypeBinding; - // TODO: SERVER-73303 refactor when v2 is enabled by default - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7292601, - str::stream() << "Unexpected encrypted payload type: " - << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindRangePayloadV2); - - auto payload = parseFromCDR<FLE2FindRangePayloadV2>(subCdr); - payloadId = payload.getPayloadId(); - firstOp = payload.getFirstOperator(); - secondOp = payload.getSecondOperator(); - - if (!payload.getPayload()) { - return; - } - - edges = std::vector<FLEFindEdgeTokenSet>(); - auto& edgesRef = edges.value(); - auto& info = payload.getPayload().value(); - - for (auto const& edge : info.getEdges()) { - auto escToken = - FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(edge.getEscDerivedToken()); - auto edcToken = - FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(edge.getEdcDerivedToken()); - auto serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - edge.getServerDerivedFromDataToken()); - edgesRef.push_back({edcToken, escToken, {}, serverDataDerivedToken}); - } - - maxCounter = info.getMaxCounter(); - return; - } - - uassert(6869501, + uassert(7292601, str::stream() << "Unexpected encrypted payload type: " << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindRangePayload); + encryptedType == EncryptedBinDataType::kFLE2FindRangePayloadV2); - auto payload = parseFromCDR<FLE2FindRangePayload>(subCdr); + auto payload = parseFromCDR<FLE2FindRangePayloadV2>(subCdr); payloadId = payload.getPayloadId(); firstOp = payload.getFirstOperator(); secondOp = payload.getSecondOperator(); @@ -5233,23 +3967,18 @@ ParsedFindRangePayload::ParsedFindRangePayload(ConstDataRange cdr) { edges = std::vector<FLEFindEdgeTokenSet>(); auto& edgesRef = edges.value(); - auto& info = payload.getPayload().value(); for (auto const& edge : info.getEdges()) { - auto escToken = FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(edge.getEscDerivedToken()); - auto eccToken = - FLETokenFromCDR<FLETokenType::ECCDerivedFromDataToken>(edge.getEccDerivedToken()); auto edcToken = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(edge.getEdcDerivedToken()); - edgesRef.push_back({edcToken, escToken, eccToken, {}}); + auto serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + edge.getServerDerivedFromDataToken()); + edgesRef.push_back({edcToken, escToken, serverDataDerivedToken}); } - serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - info.getServerEncryptionToken()); - maxCounter = info.getMaxCounter(); } @@ -5284,29 +4013,6 @@ void CompactionHelpers::validateCompactionTokens(const EncryptedFieldConfig& efc } } -std::vector<ECCDocument> CompactionHelpers::mergeECCDocuments(std::vector<ECCDocument>& unmerged) { - std::vector<ECCDocument> merged; - std::sort(unmerged.begin(), unmerged.end()); - - for (size_t i = 0; i < unmerged.size();) { - merged.push_back(unmerged[i]); - auto& last = merged.back(); - i++; - for (; i < unmerged.size() && ((last.end + 1) == unmerged[i].start); i++) { - last.end = unmerged[i].end; - } - } - return merged; -} - -uint64_t CompactionHelpers::countDeleted(const std::vector<ECCDocument>& rangeList) { - uint64_t sum = 0; - for (auto& range : rangeList) { - sum += range.end - range.start + 1; - } - return sum; -} - ConstDataRange binDataToCDR(BSONElement element) { uassert(6338501, "Expected binData BSON element", element.type() == BinData); @@ -5808,48 +4514,6 @@ OSTType_Decimal128 getTypeInfoDecimal128(Decimal128 value, return {mapping, 0, std::numeric_limits<boost::multiprecision::uint128_t>::max()}; } - -EncryptedPredicateEvaluator::EncryptedPredicateEvaluator(ConstDataRange serverToken, - int64_t contentionFactor, - std::vector<ConstDataRange> edcTokens) - : _serverToken(PrfBlockfromCDR(serverToken)), _contentionFactor(contentionFactor) { - for (auto cdr : edcTokens) { - _edcTokens.push_back(PrfBlockfromCDR(cdr)); - } - for (auto& prf : _edcTokens) { - for (auto& token : - EDCServerCollection::generateEDCTokens(ConstDataRange(prf), _contentionFactor)) { - _cachedEDCTokens.insert(std::move(token.data)); - } - } -} - -bool EncryptedPredicateEvaluator::evaluate( - Value fieldValue, - EncryptedBinDataType indexedValueType, - std::function<std::vector<EDCDerivedFromDataTokenAndContentionFactorToken>( - ConstDataRange, ConstDataRange)> decryptAndParse) const { - - if (fieldValue.getType() != BinData) { - return false; - } - - auto [subSubType, data] = fromEncryptedBinData(fieldValue); - - uassert(6672400, "Invalid encrypted indexed field", subSubType == indexedValueType); - - // Value matches if - // 1. Decrypt field is successful - // 2. EDC_u Token is in GenTokens(EDC Token, ContentionFactor) - // - auto tokens = decryptAndParse(ConstDataRange(_serverToken), data); - - return std::any_of( - std::make_move_iterator(tokens.begin()), - std::make_move_iterator(tokens.end()), - [this](auto&& token) { return _cachedEDCTokens.count(std::move(token.data)) == 1; }); -} - EncryptedPredicateEvaluatorV2::EncryptedPredicateEvaluatorV2( std::vector<ServerZerosEncryptionToken> zerosTokens) : _zerosDecryptionTokens(std::move(zerosTokens)){}; diff --git a/src/mongo/crypto/fle_crypto.h b/src/mongo/crypto/fle_crypto.h index 03636e902bc..7b49241fd54 100644 --- a/src/mongo/crypto/fle_crypto.h +++ b/src/mongo/crypto/fle_crypto.h @@ -750,20 +750,6 @@ using ContentionFactorFn = std::function<uint64_t(const FLE2EncryptionPlaceholde class FLEClientCrypto { public: - // TODO: SERVER-73303 delete v1 serialize methods when v2 is enabled by default - static FLE2FindEqualityPayload serializeFindPayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t maxContentionFactor); - - static FLE2FindRangePayload serializeFindRangePayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - const std::vector<std::string>& edges, - uint64_t maxContentionFactor, - const FLE2RangeFindSpec& spec); - - static FLE2FindRangePayload serializeFindRangeStub(const FLE2RangeFindSpec& spec); - static FLE2FindEqualityPayloadV2 serializeFindPayloadV2(FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, BSONElement element, @@ -881,24 +867,6 @@ public: ESCDerivedFromDataTokenAndContentionFactorToken esc; }; -// TODO: SERVER-73303 delete when v2 is enabled by default -struct ECOCCompactionDocument { - - bool operator==(const ECOCCompactionDocument& other) const { - return (fieldName == other.fieldName) && (esc == other.esc) && (ecc == other.ecc); - } - - template <typename H> - friend H AbslHashValue(H h, const ECOCCompactionDocument& doc) { - return H::combine(std::move(h), doc.fieldName, doc.esc, doc.ecc); - } - - // Id is not included as it unimportant - std::string fieldName; - ESCDerivedFromDataTokenAndContentionFactorToken esc; - ECCDerivedFromDataTokenAndContentionFactorToken ecc; -}; - struct ECOCCompactionDocumentV2 { bool operator==(const ECOCCompactionDocumentV2& other) const { return (fieldName == other.fieldName) && (esc == other.esc); @@ -933,68 +901,9 @@ class ECOCCollection { public: static BSONObj generateDocument(StringData fieldName, ConstDataRange payload); - // TODO: SERVER-73303 delete when v2 is enabled by default - static ECOCCompactionDocument parseAndDecrypt(const BSONObj& doc, ECOCToken token); - static ECOCCompactionDocumentV2 parseAndDecryptV2(const BSONObj& doc, ECOCToken token); }; - -/** - * Class to read/write FLE2 Equality Indexed Encrypted Values - * - * Fields are encrypted with the following: - * - * struct { - * uint8_t fle_blob_subtype = 7; - * uint8_t key_uuid[16]; - * uint8_t original_bson_type; - * ciphertext[ciphertext_length]; - * } - * - * Encrypt(ServerDataEncryptionLevel1Token, Struct(K_KeyId, v, count, d, s, c)) - * - * struct { - * uint64_t length; - * uint8_t[length] cipherText; // UserKeyId + Encrypt(K_KeyId, value), - * uint64_t counter; - * uint8_t[32] edc; // EDCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] esc; // ESCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] ecc; // ECCDerivedFromDataTokenAndContentionFactorToken - *} - * - * The specification needs to be in sync with the validation in 'bson_validate.cpp'. - */ -struct FLE2IndexedEqualityEncryptedValue { - FLE2IndexedEqualityEncryptedValue(FLE2InsertUpdatePayload payload, uint64_t counter); - - FLE2IndexedEqualityEncryptedValue(EDCDerivedFromDataTokenAndContentionFactorToken edcParam, - ESCDerivedFromDataTokenAndContentionFactorToken escParam, - ECCDerivedFromDataTokenAndContentionFactorToken eccParam, - uint64_t countParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> serializedServerValueParam); - - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); - - /** - * Read the key id from the payload. - */ - static StatusWith<UUID> readKeyId(ConstDataRange serializedServerValue); - - StatusWith<std::vector<uint8_t>> serialize(ServerDataEncryptionLevel1Token token); - - EDCDerivedFromDataTokenAndContentionFactorToken edc; - ESCDerivedFromDataTokenAndContentionFactorToken esc; - ECCDerivedFromDataTokenAndContentionFactorToken ecc; - uint64_t count; - BSONType bsonType; - UUID indexKeyId; - std::vector<uint8_t> clientEncryptedValue; -}; - /** * Class to read/write the metadata block consisting of the encrypted counter * and contention factor, the tag, and the encrypted 128-bit string of zeros. @@ -1119,32 +1028,6 @@ struct FLE2IndexedEqualityEncryptedValueV2 { FLE2TagAndEncryptedMetadataBlock metadataBlock; }; -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Class to read/write FLE2 Unindexed Encrypted Values - * - * Fields are encrypted with the following: - * - * struct { - * uint8_t fle_blob_subtype = 6; - * uint8_t key_uuid[16]; - * uint8_t original_bson_type; - * ciphertext[ciphertext_length]; - * } blob; - * - * The specification needs to be in sync with the validation in 'bson_validate.cpp'. - */ -struct FLE2UnindexedEncryptedValue { - static std::vector<uint8_t> serialize(const FLEUserKeyAndId& userKey, - const BSONElement& element); - static std::pair<BSONType, std::vector<uint8_t>> deserialize(FLEKeyVault* keyVault, - ConstDataRange blob); - - static constexpr crypto::aesMode mode = crypto::aesMode::ctr; - static constexpr EncryptedBinDataType fleType = - EncryptedBinDataType::kFLE2UnindexedEncryptedValue; - static constexpr size_t assocDataSize = sizeof(uint8_t) + sizeof(UUID) + sizeof(uint8_t); -}; /** * Class to read/write FLE2 Unindexed Encrypted Values (for protocol version 2) @@ -1191,62 +1074,6 @@ struct FLEEdgeToken { }; /** - * Class to read/write FLE2 Range Indexed Encrypted Values - * - * Fields are encrypted with the following: - * - * struct { - * uint8_t fle_blob_subtype = 9; - * uint8_t key_uuid[16]; - * uint8_t original_bson_type; - * ciphertext[ciphertext_length]; - * } - * - * Encrypt(ServerDataEncryptionLevel1Token, Struct(K_KeyId, v, edgeCount, [count, d, s, c] x - *edgeCount )) - * - * struct { - * uint64_t length; - * uint8_t[length] cipherText; // UserKeyId + Encrypt(K_KeyId, value), - * uint32_t edgeCount; - * struct { - * uint64_t counter; - * uint8_t[32] edc; // EDCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] esc; // ESCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] ecc; // ECCDerivedFromDataTokenAndContentionFactorToken - * } edges[edgeCount]; - *} - * - * The specification needs to be in sync with the validation in 'bson_validate.cpp'. - */ -struct FLE2IndexedRangeEncryptedValue { - FLE2IndexedRangeEncryptedValue(FLE2InsertUpdatePayload payload, - std::vector<uint64_t> countersParam); - - FLE2IndexedRangeEncryptedValue(std::vector<FLEEdgeToken> tokens, - std::vector<uint64_t> countersParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> serializedServerValueParam); - - static StatusWith<FLE2IndexedRangeEncryptedValue> decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); - - /** - * Read the key id from the payload. - */ - static StatusWith<UUID> readKeyId(ConstDataRange serializedServerValue); - - StatusWith<std::vector<uint8_t>> serialize(ServerDataEncryptionLevel1Token token); - - std::vector<FLEEdgeToken> tokens; - std::vector<uint64_t> counters; - BSONType bsonType; - UUID indexKeyId; - std::vector<uint8_t> clientEncryptedValue; -}; - -/** * Class to read/write QE protocol version 2 of Range Indexed * Encrypted Values. * @@ -1310,62 +1137,6 @@ struct FLE2IndexedRangeEncryptedValueV2 { std::vector<FLE2TagAndEncryptedMetadataBlock> metadataBlocks; }; -// TODO: SERVER-73303 delete when v2 is enabled by default -/* - * Shim layer for EdgeTokenSet types with different protocol versions. - */ -class VersionedEdgeTokenSet { -public: - VersionedEdgeTokenSet() = default; - VersionedEdgeTokenSet(EdgeTokenSet ets); - VersionedEdgeTokenSet(EdgeTokenSetV2 ets); - - ConstDataRange getEscDerivedToken() const; - ConstDataRange getEncryptedTokens() const; - -private: - stdx::variant<EdgeTokenSet, EdgeTokenSetV2> edgeTokenSet; -}; - -// TODO: SERVER-73303 delete when v2 is enabled by default -/* - * Shim layer for FLE2InsertUpdatePayload types with different protocol versions. - */ -class VersionedInsertUpdatePayload { -public: - VersionedInsertUpdatePayload() = default; - VersionedInsertUpdatePayload(FLE2InsertUpdatePayload iup); - VersionedInsertUpdatePayload(FLE2InsertUpdatePayloadV2 iup); - - const FLE2InsertUpdatePayload& getInsertUpdatePayloadVersion1() const; - const FLE2InsertUpdatePayloadV2& getInsertUpdatePayloadVersion2() const; - - const mongo::UUID& getIndexKeyId() const; - int getType() const; - ConstDataRange getEncryptedTokens() const; - ConstDataRange getEscDerivedToken() const; - ConstDataRange getEdcDerivedToken() const; - ConstDataRange getServerEncryptionToken() const; - const boost::optional<std::vector<VersionedEdgeTokenSet>>& getEdgeTokenSet() const; - -private: - template <class Payload> - boost::optional<std::vector<VersionedEdgeTokenSet>> convertPayloadEdgeTokenSet() { - boost::optional<std::vector<VersionedEdgeTokenSet>> converted; - auto& payload = stdx::get<Payload>(iupayload); - if (payload.getEdgeTokenSet().has_value()) { - auto& etsList = payload.getEdgeTokenSet().value(); - converted = std::vector<VersionedEdgeTokenSet>(etsList.size()); - std::transform(etsList.begin(), etsList.end(), edgeTokenSet->begin(), [](auto& ets) { - return VersionedEdgeTokenSet(ets); - }); - } - return converted; - } - stdx::variant<FLE2InsertUpdatePayload, FLE2InsertUpdatePayloadV2> iupayload; - boost::optional<std::vector<VersionedEdgeTokenSet>> edgeTokenSet; -}; - struct EDCServerPayloadInfo { static ESCDerivedFromDataTokenAndContentionFactorToken getESCToken(ConstDataRange cdr); @@ -1373,8 +1144,7 @@ struct EDCServerPayloadInfo { return payload.getEdgeTokenSet().has_value(); } - // TODO: SERVER-73303 change type to FLE2InsertUpdatePayloadV2 when v2 is enabled by default - VersionedInsertUpdatePayload payload; + FLE2InsertUpdatePayloadV2 payload; std::string fieldPathName; std::vector<uint64_t> counts; }; @@ -1429,22 +1199,14 @@ public: * Used during updates to verify that the modified document's pre-image can be * safely updated per the protocol compatibility rules. */ - static void validateModifiedDocumentCompatibility(BSONObj& originalDocument); + static void validateModifiedDocumentCompatibility(BSONObj& obj); + /** * Get information about all FLE2InsertUpdatePayload payloads */ static std::vector<EDCServerPayloadInfo> getEncryptedFieldInfo(BSONObj& obj); - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); - - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ConstDataRange token, ConstDataRange serializedServerValue); - - static StatusWith<FLE2IndexedRangeEncryptedValue> decryptAndParseRange( - ConstDataRange token, ConstDataRange serializedServerValue); - /** * Generate a search tag * @@ -1452,9 +1214,7 @@ public: */ static PrfBlock generateTag(EDCTwiceDerivedToken edcTwiceDerived, FLECounter count); static PrfBlock generateTag(const EDCServerPayloadInfo& payload); - static PrfBlock generateTag(const FLE2IndexedEqualityEncryptedValue& indexedValue); static PrfBlock generateTag(const FLEEdgeToken& token, FLECounter count); - static std::vector<PrfBlock> generateTags(const FLE2IndexedRangeEncryptedValue& indexedValue); static std::vector<PrfBlock> generateTags(const EDCServerPayloadInfo& rangePayload); /** @@ -1483,17 +1243,6 @@ public: static BSONObj finalizeForUpdate(const BSONObj& doc, const std::vector<EDCServerPayloadInfo>& serverPayload); - // TODO: SERVER-73303 remove once v2 is enabled by default - /** - * Generate an update modifier document with $pull to remove stale tags. - * - * Generates: - * - * { $pull : {__safeContent__ : {$in : [tag..] } } } - */ - static BSONObj generateUpdateToRemoveTags(const std::vector<EDCIndexedFields>& removedFields, - const StringMap<FLEDeleteToken>& tokenMap); - /** * Generate an update modifier document with $pull to remove stale tags. * @@ -1549,27 +1298,11 @@ public: static BSONObj encryptionInformationSerialize(const NamespaceString& nss, const BSONObj& encryptedFields); - // TODO: SERVER-73303 remove when v2 is enabled by default - /** - * Serialize EncryptionInformation with EncryptionInformation.schema and a map of delete tokens - * for each field in EncryptedFieldConfig. - */ - static BSONObj encryptionInformationSerializeForDelete(const NamespaceString& nss, - const EncryptedFieldConfig& ef, - FLEKeyVault* keyVault); - /** * Get a schema from EncryptionInformation and ensure the esc/ecc/ecoc are setup correctly. */ static EncryptedFieldConfig getAndValidateSchema(const NamespaceString& nss, const EncryptionInformation& ei); - - // TODO: SERVER-73303 remove when v2 is enabled by default - /** - * Get a set of delete tokens for a given nss from EncryptionInformation. - */ - static StringMap<FLEDeleteToken> getDeleteTokens(const NamespaceString& nss, - const EncryptionInformation& ei); }; /** @@ -1595,25 +1328,6 @@ public: * in the encrypted field config */ static void validateCompactionTokens(const EncryptedFieldConfig& efc, BSONObj compactionTokens); - - // TODO: SERVER-73303 delete when v2 is enabled by default - /** - * Merges the list of ECCDocuments so that entries whose tuple values are - * adjacent to each other are combined into a single entry. For example, - * the input [ (1,3), (11,11), (7,9), (4,6) ] outputs [ (1,9), (11,11) ]. - * Assumes none of the input entries overlap with each other. - * - * This will sort the input unmerged list as a side-effect. - */ - static std::vector<ECCDocument> mergeECCDocuments(std::vector<ECCDocument>& unmerged); - - // TODO: SERVER-73303 delete when v2 is enabled by default - /** - * Given a list of ECCDocument, where each document is a range of - * deleted positions, this calculates the total number of deleted - * positions. - */ - static uint64_t countDeleted(const std::vector<ECCDocument>& rangeList); }; /** @@ -1626,10 +1340,7 @@ std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(Cons struct ParsedFindEqualityPayload { ESCDerivedFromDataToken escToken; - // TODO: SERVER-73303 remove eccToken and serverToken when v2 is enabled by default - ECCDerivedFromDataToken eccToken; EDCDerivedFromDataToken edcToken; - boost::optional<ServerDataEncryptionLevel1Token> serverToken; boost::optional<std::int64_t> maxCounter; // v2 fields @@ -1726,18 +1437,12 @@ struct FLEFindEdgeTokenSet { EDCDerivedFromDataToken edc; ESCDerivedFromDataToken esc; - // TODO: SERVER-73303 remove ecc field when v2 is enabled by default - ECCDerivedFromDataToken ecc; - ServerDerivedFromDataToken server; }; struct ParsedFindRangePayload { boost::optional<std::vector<FLEFindEdgeTokenSet>> edges; - // TODO: SERVER-73303 remove serverToken when v2 is enabled by default - ServerDataEncryptionLevel1Token serverToken; - Fle2RangeOperator firstOp; boost::optional<Fle2RangeOperator> secondOp; std::int32_t payloadId{}; diff --git a/src/mongo/crypto/fle_crypto_predicate.h b/src/mongo/crypto/fle_crypto_predicate.h index be4856edb72..79f6c2037cc 100644 --- a/src/mongo/crypto/fle_crypto_predicate.h +++ b/src/mongo/crypto/fle_crypto_predicate.h @@ -51,45 +51,6 @@ namespace mongo { -// TODO: SERVER-73303 delete class when v2 is enabled by default -class EncryptedPredicateEvaluator { -public: - EncryptedPredicateEvaluator(ConstDataRange serverToken, - int64_t contentionFactor, - std::vector<ConstDataRange> edcTokens); - - EncryptedPredicateEvaluator() {} - - /** - * Evaluates an encrypted predicate (equality or range), as defined by a contention factor along - * with a list of edc tokens, over an encrypted value. - * - * Returns a boolean indicator. - */ - bool evaluate(Value fieldValue, - EncryptedBinDataType indexedValueType, - std::function<std::vector<EDCDerivedFromDataTokenAndContentionFactorToken>( - ConstDataRange, ConstDataRange)> decryptAndParse) const; - - std::vector<PrfBlock> edcTokens() const { - return _edcTokens; - } - - PrfBlock serverToken() const { - return _serverToken; - } - - int64_t contentionFactor() const { - return _contentionFactor; - } - -private: - PrfBlock _serverToken; - std::vector<PrfBlock> _edcTokens; - int64_t _contentionFactor{0}; - stdx::unordered_set<PrfBlock> _cachedEDCTokens; -}; - class EncryptedPredicateEvaluatorV2 { public: EncryptedPredicateEvaluatorV2(std::vector<ServerZerosEncryptionToken> zerosTokens); diff --git a/src/mongo/crypto/fle_crypto_test.cpp b/src/mongo/crypto/fle_crypto_test.cpp index 8514057d678..15acde90f69 100644 --- a/src/mongo/crypto/fle_crypto_test.cpp +++ b/src/mongo/crypto/fle_crypto_test.cpp @@ -365,16 +365,6 @@ TEST(FLETokens, TestVectorUnindexedValueDecryption) { // Unindexed field decryption // Encryption can not be generated using test vectors because IV is random TestKeyVault keyVault; - { - const std::string uxCiphertext = hexblob::decode( - "06ABCDEFAB12349876123412345678901202F2CE7FDD0DECD5442CC98C10B9138741785173E323132982740496768877A3BA46581CED4A34031B1174B5C524C15BAAE687F88C29FC71F40A32BCD53D63CDA0A6646E8677E167BB3A933529F5B519CFE255BBC323D943B4F105"_sd); - auto [uxBsonType, uxPlaintext] = - FLE2UnindexedEncryptedValue::deserialize(&keyVault, ConstDataRange(uxCiphertext)); - ASSERT_EQUALS(uxBsonType, BSONType::String); - ASSERT_EQUALS( - hexblob::encode(uxPlaintext.data(), uxPlaintext.size()), - "260000004C6F7279207761732061206D6F75736520696E2061206269672062726F776E20686F75736500"); - } { const std::string uxCiphertext = hexblob::decode( @@ -388,36 +378,6 @@ TEST(FLETokens, TestVectorUnindexedValueDecryption) { } } -TEST(FLETokens, TestVectorIndexedValueDecryption) { - // Equality indexed field decryption - // Encryption can not be generated using test vectors because IV is random - - const std::string ixCiphertext = hexblob::decode( - "000000000000000000000000000000000297044B8E1B5CF4F9052EDB50236A343597C418A74352F98357A77E0D4299C04151CBEC24A5D5349A5A5EAA1FE334154FEEB6C8E7BD636089904F76950B2184D146792CBDF9179FFEDDB7D90FC257BB13DCB3E731182A447E2EF1BE7A2AF13DC9362701BABDE0B5E78CF4A92227D5B5D1E1556E75BAB5B4E9F5CEFEA3BA3E3D5D31D11B20619437A30550EFF5B602357567CF05058E4F84A103293F70302F3A50667642DD0325D194A197"_sd); - ServerDataEncryptionLevel1Token serverEncryptToken( - decodePrf("EB9A73F7912D86A4297E81D2F675AF742874E4057E3A890FEC651A23EEE3F3EC"_sd)); - - auto swServerPayload = FLE2IndexedEqualityEncryptedValue::decryptAndParse( - serverEncryptToken, ConstDataRange(ixCiphertext)); - ASSERT_OK(swServerPayload.getStatus()); - - auto cdrEqualHex = [](ConstDataRange cdr, const StringData hex) -> bool { - const std::string s = hexblob::decode(hex); - return cdr.length() == s.size() && std::equal(s.begin(), s.end(), cdr.data<char>()); - }; - - auto sp = swServerPayload.getValue(); - ASSERT(cdrEqualHex(sp.edc.toCDR(), - "97C8DFE394D80A4EE335E3F9FDC024D18BE4B92F9444FCA316FF9896D7BF455D"_sd)); - ASSERT(cdrEqualHex(sp.esc.toCDR(), - "EBB22F74BE0FA4AD863188D3F33AF0B95CB4CA4ED0091E1A43513DB20E9D59AE"_sd)); - ASSERT(cdrEqualHex(sp.ecc.toCDR(), - "A1DF0BB04C977BD4BC0B487FFFD2E3BBB96078354DE9F204EE5872BB10F01971"_sd)); - ASSERT_EQ(sp.count, 123456); - ASSERT(cdrEqualHex( - sp.clientEncryptedValue, - "260000004C6F7279207761732061206D6F75736520696E2061206269672062726F776E20686F75736500"_sd)); -} TEST(FLETokens, TestVectorESCCollectionDecryptDocument) { ESCTwiceDerivedTagToken escTwiceTag( @@ -1969,163 +1929,6 @@ TEST(FLE_EDC, ServerSide_Payloads_V2_IsValidZerosBlob) { ASSERT_FALSE(FLE2TagAndEncryptedMetadataBlock::isValidZerosBlob(zeros)); } -TEST(FLE_EDC, ServerSide_Payloads) { - TestKeyVault keyVault; - - auto doc = BSON("sample" << 123456); - auto element = doc.firstElement(); - - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(getIndexKey()); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(getIndexKey()); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - FLECounter counter = 0; - - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, counter); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, counter); - - FLE2InsertUpdatePayload iupayload; - - - iupayload.setEdcDerivedToken(edcDatakey.toCDR()); - iupayload.setEscDerivedToken(escDatakey.toCDR()); - iupayload.setEccDerivedToken(eccDatakey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - iupayload.setValue(value); - iupayload.setType(element.type()); - - FLE2IndexedEqualityEncryptedValue serverPayload(iupayload, 123456); - - auto swBuf = serverPayload.serialize(serverEncryptToken); - ASSERT_OK(swBuf.getStatus()); - - auto swServerPayload = - FLE2IndexedEqualityEncryptedValue::decryptAndParse(serverEncryptToken, swBuf.getValue()); - - ASSERT_OK(swServerPayload.getStatus()); - auto sp = swServerPayload.getValue(); - ASSERT_EQ(sp.edc, serverPayload.edc); - ASSERT_EQ(sp.esc, serverPayload.esc); - ASSERT_EQ(sp.ecc, serverPayload.ecc); - ASSERT_EQ(sp.count, serverPayload.count); - ASSERT(sp.clientEncryptedValue == serverPayload.clientEncryptedValue); - ASSERT_EQ(serverPayload.clientEncryptedValue.size(), value.length()); - ASSERT(std::equal(serverPayload.clientEncryptedValue.begin(), - serverPayload.clientEncryptedValue.end(), - value.data<uint8_t>())); -} - -TEST(FLE_EDC, ServerSide_Range_Payloads) { - TestKeyVault keyVault; - - auto doc = BSON("sample" << 3); - auto element = doc.firstElement(); - - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(getIndexKey()); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(getIndexKey()); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - FLECounter counter = 0; - - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, counter); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, counter); - - FLE2InsertUpdatePayload iupayload; - - - iupayload.setEdcDerivedToken(edcDatakey.toCDR()); - iupayload.setEscDerivedToken(escDatakey.toCDR()); - iupayload.setEccDerivedToken(eccDatakey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - iupayload.setValue(value); - iupayload.setType(element.type()); - - std::vector<EdgeTokenSet> tokens; - EdgeTokenSet ets; - ets.setEdcDerivedToken(edcDatakey.toCDR()); - ets.setEscDerivedToken(escDatakey.toCDR()); - ets.setEccDerivedToken(eccDatakey.toCDR()); - ets.setEncryptedTokens(swEncryptedTokens.getValue()); - - tokens.push_back(ets); - tokens.push_back(ets); - - iupayload.setEdgeTokenSet(tokens); - - FLE2IndexedRangeEncryptedValue serverPayload(iupayload, {123456, 123456}); - - auto swBuf = serverPayload.serialize(serverEncryptToken); - ASSERT_OK(swBuf.getStatus()); - - auto swServerPayload = - FLE2IndexedRangeEncryptedValue::decryptAndParse(serverEncryptToken, swBuf.getValue()); - - ASSERT_OK(swServerPayload.getStatus()); - auto sp = swServerPayload.getValue(); - ASSERT_EQ(sp.tokens.size(), 2); - for (size_t i = 0; i < sp.tokens.size(); i++) { - auto ets = sp.tokens[i]; - auto rhs = serverPayload.tokens[i]; - ASSERT_EQ(ets.edc, rhs.edc); - ASSERT_EQ(ets.esc, rhs.esc); - ASSERT_EQ(ets.ecc, rhs.ecc); - ASSERT_EQ(sp.counters[i], serverPayload.counters[i]); - } - - ASSERT(sp.clientEncryptedValue == serverPayload.clientEncryptedValue); - ASSERT_EQ(serverPayload.clientEncryptedValue.size(), value.length()); - ASSERT(std::equal(serverPayload.clientEncryptedValue.begin(), - serverPayload.clientEncryptedValue.end(), - value.data<uint8_t>())); -} TEST(FLE_EDC, ServerSide_Range_Payloads_V2) { TestKeyVault keyVault; @@ -2411,7 +2214,6 @@ TEST(EncryptionInformation, BadSchema) { } TEST(EncryptionInformation, MissingStateCollection) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); { @@ -2509,58 +2311,6 @@ TEST(IndexedFields, DuplicateIndexKeyIds) { ASSERT_THROWS_CODE(encryptDocument(builder.obj(), &keyVault), DBException, 6371407); } -TEST(DeleteTokens, Basic) { - TestKeyVault keyVault; - NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); - EncryptedFieldConfig efc = getTestEncryptedFieldConfig(); - - auto obj = - EncryptionInformationHelpers::encryptionInformationSerializeForDelete(ns, efc, &keyVault); - - std::cout << "Tokens" << obj << std::endl; -} - -TEST(DeleteTokens, Fetch) { - TestKeyVault keyVault; - NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); - EncryptedFieldConfig efc = getTestEncryptedFieldConfig(); - - auto obj = - EncryptionInformationHelpers::encryptionInformationSerializeForDelete(ns, efc, &keyVault); - - auto tokenMap = EncryptionInformationHelpers::getDeleteTokens( - ns, EncryptionInformation::parse(IDLParserContext("foo"), obj)); - - ASSERT_EQ(tokenMap.size(), 2); - - ASSERT(tokenMap.contains("nested.encrypted")); - ASSERT(tokenMap.contains("encrypted")); -} - -TEST(DeleteTokens, CorruptDelete) { - TestKeyVault keyVault; - NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); - EncryptedFieldConfig efc = getTestEncryptedFieldConfig(); - - EncryptionInformation ei; - ei.setType(1); - - ei.setSchema(BSON(ns.toString() << efc.toBSON())); - - // Missing Delete tokens - ASSERT_THROWS_CODE(EncryptionInformationHelpers::getDeleteTokens(ns, ei), DBException, 6371308); - - // NSS map is not an object - ei.setDeleteTokens(BSON(ns.toString() << "str")); - - ASSERT_THROWS_CODE(EncryptionInformationHelpers::getDeleteTokens(ns, ei), DBException, 6371309); - - // Tokens is not a map - ei.setDeleteTokens(BSON(ns.toString() << BSON("a" - << "b"))); - - ASSERT_THROWS_CODE(EncryptionInformationHelpers::getDeleteTokens(ns, ei), DBException, 6371310); -} // Verify we can compare two list of tags correctly TEST(TagDelta, Basic) { @@ -2650,20 +2400,6 @@ TEST(EDC, UnindexedEncryptDecrypt) { std::vector<uint8_t>(element.value(), element.value() + element.valuesize()); { - auto blob = FLE2UnindexedEncryptedValue::serialize(userKey, element); - ASSERT_EQ(blob[0], 6); - - // assert length of ciphertext (including HMAC & IV) is consistent with CTR mode - auto cipherTextLen = blob.size() - FLE2UnindexedEncryptedValue::assocDataSize; - ASSERT_EQ(cipherTextLen, - crypto::fle2AeadCipherOutputLength(elementData.size(), crypto::aesMode::ctr)); - - auto [type, plainText] = FLE2UnindexedEncryptedValue::deserialize(&keyVault, {blob}); - ASSERT_EQ(type, element.type()); - ASSERT_TRUE( - std::equal(plainText.begin(), plainText.end(), elementData.begin(), elementData.end())); - } - { auto blob = FLE2UnindexedEncryptedValueV2::serialize(userKey, element); ASSERT_EQ(blob[0], 16); @@ -3124,43 +2860,6 @@ std::vector<ECCDocument> pairsToECCDocuments( return output; } -TEST(CompactionHelpersTest, mergeECCDocumentsTest) { - std::vector<ECCDocument> input, output, expected; - - // Test empty input - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output.empty()); - - // Test single pair - input = pairsToECCDocuments({{15, 20}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output == input); - - // Test input with no gaps - input = pairsToECCDocuments({{15, 20}, {13, 13}, {1, 6}, {7, 12}, {14, 14}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT_EQ(output.size(), 1); - ASSERT_EQ(output.front().start, 1); - ASSERT_EQ(output.front().end, 20); - - // Test input with gaps; nothing is merged - input = pairsToECCDocuments({{5, 5}, {12, 16}, {9, 9}, {23, 45}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output == input); - - // Test input with gaps; at least one merged - input = pairsToECCDocuments({{5, 5}, {12, 16}, {6, 9}, {17, 23}, {45, 45}}); - expected = pairsToECCDocuments({{5, 9}, {12, 23}, {45, 45}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output == expected); -} - -TEST(CompactionHelpersTest, countDeletedTest) { - ASSERT_EQ(CompactionHelpers::countDeleted({}), 0); - - auto input = pairsToECCDocuments({{15, 20}, {13, 13}, {1, 6}, {7, 12}, {14, 14}}); - ASSERT_EQ(CompactionHelpers::countDeleted(input), 20); -} TEST(EDCServerCollectionTest, GenerateEDCTokens) { @@ -3183,8 +2882,6 @@ TEST(EDCServerCollectionTest, GenerateEDCTokens) { } TEST(EDCServerCollectionTest, ValidateModifiedDocumentCompatibility) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); - std::vector<uint8_t> blob; std::vector<EncryptedBinDataType> badTypes = { EncryptedBinDataType::kFLE2EqualityIndexedValue, diff --git a/src/mongo/crypto/fle_field_schema.idl b/src/mongo/crypto/fle_field_schema.idl index 8e1d03c6cc8..d8a6f1fdfaf 100644 --- a/src/mongo/crypto/fle_field_schema.idl +++ b/src/mongo/crypto/fle_field_schema.idl @@ -187,27 +187,6 @@ structs: optional: true validator: { gte: 1, lte: 4 } - EdgeTokenSet: - description: "Payload of an indexed field to insert or update" - strict: true - fields: - d: - description: "EDCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: eccDerivedToken - p: - description: "Encrypted tokens" - type: bindata_generic - cpp_name: encryptedTokens - EdgeTokenSetV2: description: "Payload of an indexed field to insert or update. Version 2" strict: true @@ -229,48 +208,6 @@ structs: type: bindata_generic cpp_name: encryptedTokens - FLE2InsertUpdatePayload: - description: "Payload of an indexed field to insert or update" - strict: true - fields: - d: - description: "EDCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: eccDerivedToken - p: - description: "Encrypted tokens" - type: bindata_generic - cpp_name: encryptedTokens - u: - description: "Index KeyId" - type: uuid - cpp_name: indexKeyId - t: - description: "Encrypted type" - type: safeInt - cpp_name: type - v: - description: "Encrypted value" - type: bindata_generic - cpp_name: value - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - g: - description: "Array of Edges" - type: array<EdgeTokenSet> - cpp_name: edgeTokenSet - optional: true - FLE2InsertUpdatePayloadV2: description: "Payload of an indexed field to insert or update. Version 2" strict: true @@ -318,46 +255,6 @@ structs: cpp_name: edgeTokenSet optional: true - FLE2DeletePayload: - description: "Payload of an indexed field to delete" - strict: true - fields: - o: - description: "ECOCToken" - type: bindata_generic - cpp_name: ecocToken - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - - FLE2FindEqualityPayload: - description: "Payload for an equality find" - strict: true - fields: - d: - description: "EDCDerivedFromDataToken" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataToken" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataToken" - type: bindata_generic - cpp_name: eccDerivedToken - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - optional: true # For backwards compat, make this optional - cm: - description: "Queryable Encryption max counter" - type: long - cpp_name: maxCounter - optional: true - FLE2FindEqualityPayloadV2: description: "Payload for an equality find. Version 2" strict: true @@ -380,23 +277,6 @@ structs: cpp_name: maxCounter optional: true - EdgeFindTokenSet: - description: "Payload of an edge to query for" - strict: true - fields: - d: - description: "EDCDerivedFromDataToken" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataToken" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataToken" - type: bindata_generic - cpp_name: eccDerivedToken - EdgeFindTokenSetV2: description: "Payload of an edge to query for. Version 2" strict: true @@ -414,23 +294,6 @@ structs: type: bindata_generic cpp_name: serverDerivedFromDataToken - FLE2FindRangePayloadEdgesInfo: - description: "Token information for find range payload." - strict: true - fields: - g: - description: "Array of Edges" - type: array<EdgeFindTokenSet> - cpp_name: edges - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - cm: - description: "Queryable Encryption max counter" - type: long - cpp_name: maxCounter - FLE2FindRangePayloadEdgesInfoV2: description: "Token information for find range payload." strict: true @@ -444,27 +307,6 @@ structs: type: long cpp_name: maxCounter - FLE2FindRangePayload: - description: "Payload for a range find" - strict: true - fields: - payload: - description: "Token information for a find range payload" - type: FLE2FindRangePayloadEdgesInfo - optional: true - payloadId: - description: "Id of payload - must be paired with another payload" - type: safeInt - optional: false - firstOperator: - description: "First query operator for which this payload was generated." - type: Fle2RangeOperator - optional: false - secondOperator: - description: "Second query operator for which this payload was generated. Only populated for two-sided ranges." - type: Fle2RangeOperator - optional: true - FLE2FindRangePayloadV2: description: "Payload for a range find. Version 2" strict: true diff --git a/src/mongo/crypto/fle_stats.h b/src/mongo/crypto/fle_stats.h index 6b94c2427a9..39eec428097 100644 --- a/src/mongo/crypto/fle_stats.h +++ b/src/mongo/crypto/fle_stats.h @@ -101,13 +101,6 @@ public: _hasStats.store(true); accumulateStats(_compactStats.getEsc(), stats.getEsc()); accumulateStats(_compactStats.getEcoc(), stats.getEcoc()); - - if (stats.getEcc().has_value()) { - if (!_compactStats.getEcc().has_value()) { - _compactStats.setEcc(ECStats{}); - } - accumulateStats(_compactStats.getEcc().value(), stats.getEcc().value()); - } } private: diff --git a/src/mongo/crypto/fle_stats.idl b/src/mongo/crypto/fle_stats.idl index fdd6026657a..485a59f8f50 100644 --- a/src/mongo/crypto/fle_stats.idl +++ b/src/mongo/crypto/fle_stats.idl @@ -60,10 +60,10 @@ structs: default: 0 CompactStats: - description: "Stats about records in ECC, ECOC, and ESC compact touched" + description: "Stats about records in ECOC, and ESC compact touched" fields: ecoc: ECOCStats - ecc: # TODO: SERVER-73303 remove ecc when v2 is enabled by default + ecc: # TODO: SERVER-68373 remove ecc when 7.0 becomes last LTS type: ECStats optional: true esc: ECStats diff --git a/src/mongo/crypto/fle_tags.cpp b/src/mongo/crypto/fle_tags.cpp index c1c07e391b8..ffd901e3567 100644 --- a/src/mongo/crypto/fle_tags.cpp +++ b/src/mongo/crypto/fle_tags.cpp @@ -95,123 +95,6 @@ size_t sizeArrayElementsMemory(size_t tagCount) { return size; } - -// TODO: SERVER-73303 remove when v2 is enabled by default -// The algorithm for constructing a list of tags matching an equality predicate on an encrypted -// field is as follows: -// -// (1) Query ESC to obtain the counter value (n) after the most recent insert. -// (2) Query ECC for a null document. -// (a) A null document means there has been at least one compaction of ECC. -// (b) No null document means there has not been a compaction. Therefore we have to check all -// the tags from 1..n to see if they have been deleted. -// (3) Return the surviving set of tags from 1..n (encrypted). -// -// Given: -// s: ESCDerivedFromDataToken -// c: ECCDerivedFromDataToken -// d: EDCDerivedFromDataToken -// Do: -// n = ESC::emuBinary(s) -// deletedTags = [] -// pos = ECC::nullDocument(c) ? ECC::nullDocument(c).position : 1 -// while true { -// if (doc = ECC::getDocument(c, pos); doc != null) { -// deletedTags = [doc | deletedTags] -// } else { -// break -// } -// pos++ -// } -// return [EDC::encrypt(i) | i in [1..n] where i not in deletedTags] -std::vector<PrfBlock> readTagsWithContention(const FLEStateCollectionReader& esc, - const FLEStateCollectionReader& ecc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - uint64_t cf, - size_t memoryLimit, - std::vector<PrfBlock>&& binaryTags) { - - auto escTok = DerivedToken::generateESCDerivedFromDataTokenAndContentionFactorToken(s, cf); - auto escTag = TwiceDerived::generateESCTwiceDerivedTagToken(escTok); - auto escVal = TwiceDerived::generateESCTwiceDerivedValueToken(escTok); - - auto eccTok = DerivedToken::generateECCDerivedFromDataTokenAndContentionFactorToken(c, cf); - auto eccTag = TwiceDerived::generateECCTwiceDerivedTagToken(eccTok); - auto eccVal = TwiceDerived::generateECCTwiceDerivedValueToken(eccTok); - - auto edcTok = DerivedToken::generateEDCDerivedFromDataTokenAndContentionFactorToken(d, cf); - auto edcTag = TwiceDerived::generateEDCTwiceDerivedToken(edcTok); - - // (1) Query ESC for the counter value after the most recent insert. - // 0 => 0 inserts for this field value pair. - // n => n inserts for this field value pair. - // none => compaction => query ESC for null document to find # of inserts. - auto insertCounter = ESCCollection::emuBinary(esc, escTag, escVal); - if (insertCounter && insertCounter.value() == 0) { - return std::move(binaryTags); - } - - auto numInserts = insertCounter - ? uassertStatusOK( - ESCCollection::decryptDocument( - escVal, esc.getById(ESCCollection::generateId(escTag, insertCounter)))) - .count - : uassertStatusOK(ESCCollection::decryptNullDocument( - escVal, esc.getById(ESCCollection::generateId(escTag, boost::none)))) - .count; - - // (2) Query ECC for a null document. - auto eccNullDoc = ecc.getById(ECCCollection::generateId(eccTag, boost::none)); - auto pos = eccNullDoc.isEmpty() - ? 1 - : uassertStatusOK(ECCCollection::decryptNullDocument(eccVal, eccNullDoc)).position + 2; - - std::vector<ECCDocument> deletes; - - // (2) Search ECC for deleted tag(counter) values. - while (true) { - auto eccObj = ecc.getById(ECCCollection::generateId(eccTag, pos)); - if (eccObj.isEmpty()) { - break; - } - auto eccDoc = uassertStatusOK(ECCCollection::decryptDocument(eccVal, eccObj)); - // Compaction placeholders only present for positive contention factors (cm). - if (eccDoc.valueType == ECCValueType::kCompactionPlaceholder) { - break; - } - // Note, in the worst case where no compactions have occurred, the deletes vector will grow - // proportionally to the number of deletes. This is not likely to present a significant - // problem but we should still track the memory we consume here. - deletes.emplace_back(eccDoc); - pos++; - } - - std::sort(deletes.begin(), deletes.end()); - - auto numDeletes = std::accumulate(deletes.begin(), deletes.end(), 0, [](auto acc, auto eccDoc) { - return acc + eccDoc.end - eccDoc.start + 1; - }); - auto cumTagCount = binaryTags.size() + numInserts - numDeletes; - - verifyTagsWillFit(cumTagCount, memoryLimit); - - for (uint64_t i = 1; i <= numInserts; i++) { - if (auto it = std::lower_bound( - deletes.begin(), - deletes.end(), - i, - [](ECCDocument& eccDoc, uint64_t tag) { return eccDoc.end < tag; }); - it != deletes.end() && it->start <= i && i <= it->end) { - continue; - } - // (3) Return the surviving set of tags from 1..n (encrypted). - binaryTags.emplace_back(EDCServerCollection::generateTag(edcTag, i)); - } - return std::move(binaryTags); -} - // A positive contention factor (cm) means we must run the above algorithm (cm) times. std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc, @@ -268,37 +151,4 @@ std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, return binaryTags; } - -// TODO: SERVER-73303 remove when v2 is enabled by default -// A positive contention factor (cm) means we must run the above algorithm (cm) times. -std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - boost::optional<int64_t> cm) { - - // The output of readTags will be used as the argument to a $in expression, so make sure we - // don't exceed the configured memory limit. - auto memoryLimit = static_cast<size_t>(internalQueryFLERewriteMemoryLimit.load()); - auto contentionMax = cm.value_or(0); - std::vector<PrfBlock> binaryTags; - - auto makeCollectionReader = [](FLETagQueryInterface* queryImpl, const NamespaceString& nss) { - auto docCount = queryImpl->countDocuments(nss); - return TxnCollectionReader(docCount, queryImpl, nss); - }; - - // Construct FLE rewriter from the transaction client and encryptionInformation. - auto esc = makeCollectionReader(queryImpl, nssEsc); - auto ecc = makeCollectionReader(queryImpl, nssEcc); - - for (auto i = 0; i <= contentionMax; i++) { - binaryTags = - readTagsWithContention(esc, ecc, s, c, d, i, memoryLimit, std::move(binaryTags)); - } - - return binaryTags; -} } // namespace mongo::fle diff --git a/src/mongo/crypto/fle_tags.h b/src/mongo/crypto/fle_tags.h index 9597924b6ea..ac69de0350a 100644 --- a/src/mongo/crypto/fle_tags.h +++ b/src/mongo/crypto/fle_tags.h @@ -40,32 +40,9 @@ class FLETagQueryInterface; namespace mongo::fle { /** - * Read a list of binary tags given ESC, ECC, and EDC derived tokens and a specific contention + * Read a list of binary tags given ESC and and EDC derived tokens and a maximum contention * factor. */ -std::vector<PrfBlock> readTagsWithContention(const FLEStateCollectionReader& esc, - const FLEStateCollectionReader& ecc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - uint64_t contentionFactor, - size_t memoryLimit, - std::vector<PrfBlock>&& binaryTags); - -/** - * Read a list of binary tags given ESC, ECC, and EDC derived tokens and a maximum contention - * factor. - * - * TODO: SERVER-73303 remove when v2 is enabled by default - */ -std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - boost::optional<int64_t> cm); - std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc, ESCDerivedFromDataToken s, diff --git a/src/mongo/db/catalog/collection_options.cpp b/src/mongo/db/catalog/collection_options.cpp index d4820373455..86ccb5e4a6f 100644 --- a/src/mongo/db/catalog/collection_options.cpp +++ b/src/mongo/db/catalog/collection_options.cpp @@ -85,14 +85,6 @@ void setEncryptedDefaultEncryptedCollectionNames(const NamespaceString& ns, config->setEscCollection(StringData(prefix + ".esc")); } - // (Ignore FCV check): This feature flag check will be deleted before release. - // TODO SERVER-73303: delete when v2 is enabled. - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabledAndIgnoreFCVUnsafe()) { - if (!config->getEccCollection()) { - config->setEccCollection(StringData(prefix + ".ecc")); - } - } - if (!config->getEcocCollection()) { config->setEcocCollection(StringData(prefix + ".ecoc")); } diff --git a/src/mongo/db/commands/fle2_compact.cpp b/src/mongo/db/commands/fle2_compact.cpp index d1f00e68113..f637e1c8dc8 100644 --- a/src/mongo/db/commands/fle2_compact.cpp +++ b/src/mongo/db/commands/fle2_compact.cpp @@ -52,11 +52,6 @@ MONGO_FAIL_POINT_DEFINE(fleCompactFailBeforeECOCRead); MONGO_FAIL_POINT_DEFINE(fleCompactHangBeforeESCAnchorInsert); -// TODO: SERVER-73303 delete the below failpoints when v2 is enabled by default -MONGO_FAIL_POINT_DEFINE(fleCompactHangBeforeESCPlaceholderInsert); -MONGO_FAIL_POINT_DEFINE(fleCompactHangAfterESCPlaceholderInsert); -MONGO_FAIL_POINT_DEFINE(fleCompactHangBeforeECCPlaceholderInsert); -MONGO_FAIL_POINT_DEFINE(fleCompactHangAfterECCPlaceholderInsert); namespace mongo { namespace { @@ -138,300 +133,6 @@ private: const NamespaceString& _nss; mutable CompactStatsCounter<StatsType> _stats; }; - -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Deletes an entry at the given position from FLECollection, using - * the TagToken to generate the _id value for the delete query. - */ -template <typename FLECollection, typename TagToken> -void deleteDocumentByPos(FLEQueryInterface* queryImpl, - const NamespaceString& nss, - uint64_t pos, - const TagToken& tagToken, - ECStats* stats) { - CompactStatsCounter<ECStats> statsCtr(stats); - - write_ops::DeleteOpEntry deleteEntry; - auto block = FLECollection::generateId(tagToken, pos); - deleteEntry.setMulti(false); - deleteEntry.setQ(BSON("_id" << BSONBinData(block.data(), block.size(), BinDataGeneral))); - write_ops::DeleteCommandRequest deleteRequest(nss, {std::move(deleteEntry)}); - auto [deleteReply, deletedDoc] = - queryImpl->deleteWithPreimage(nss, EncryptionInformation(BSONObj()), deleteRequest); - - if (deletedDoc.isEmpty()) { - // nothing was deleted - return; - } - checkWriteErrors(deleteReply); - statsCtr.addDeletes(1); -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Inserts or updates a null document in FLECollection. - * The newNullDoc must contain the _id of the null document to update. - */ -void upsertNullDocument(FLEQueryInterface* queryImpl, - bool hasNullDoc, - BSONObj newNullDoc, - const NamespaceString& nss, - ECStats* stats) { - CompactStatsCounter<ECStats> statsCtr(stats); - if (hasNullDoc) { - // update the null doc with a replacement modification - write_ops::UpdateOpEntry updateEntry; - updateEntry.setMulti(false); - updateEntry.setUpsert(false); - updateEntry.setQ(newNullDoc.getField("_id").wrap()); - updateEntry.setU(mongo::write_ops::UpdateModification( - newNullDoc, write_ops::UpdateModification::ReplacementTag{})); - write_ops::UpdateCommandRequest updateRequest(nss, {std::move(updateEntry)}); - auto [reply, originalDoc] = - queryImpl->updateWithPreimage(nss, EncryptionInformation(BSONObj()), updateRequest); - checkWriteErrors(reply); - if (!originalDoc.isEmpty()) { - statsCtr.addUpdates(1); - } - } else { - // insert the null doc; translate duplicate key error to a FLE contention error - StmtId stmtId = kUninitializedStmtId; - auto reply = uassertStatusOK(queryImpl->insertDocuments(nss, {newNullDoc}, &stmtId, true)); - checkWriteErrors(reply); - statsCtr.addInserts(1); - } -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Deletes a document at the specified position from the ESC - */ -void deleteESCDocument(FLEQueryInterface* queryImpl, - const NamespaceString& nss, - uint64_t pos, - const ESCTwiceDerivedTagToken& tagToken, - ECStats* escStats) { - deleteDocumentByPos<ESCCollection, ESCTwiceDerivedTagToken>( - queryImpl, nss, pos, tagToken, escStats); -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Deletes a document at the specified position from the ECC - */ -void deleteECCDocument(FLEQueryInterface* queryImpl, - const NamespaceString& nss, - uint64_t pos, - const ECCTwiceDerivedTagToken& tagToken, - ECStats* eccStats) { - deleteDocumentByPos<ECCCollection, ECCTwiceDerivedTagToken>( - queryImpl, nss, pos, tagToken, eccStats); -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Result of preparing the ESC collection for a single field/value pair - * before compaction. - */ -struct ESCPreCompactState { - // total insertions of this field/value pair into EDC - uint64_t count{0}; - // position of the lowest entry - uint64_t ipos{0}; - // position of the highest entry - uint64_t pos{0}; -}; - -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Finds the upper and lower bound positions, and the current counter - * value from the ESC collection for the given twice-derived tokens, - * and inserts the compaction placeholder document. - */ -ESCPreCompactState prepareESCForCompaction(FLEQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const ESCTwiceDerivedTagToken& tagToken, - const ESCTwiceDerivedValueToken& valueToken, - ECStats* escStats) { - CompactStatsCounter<ECStats> stats(escStats); - - TxnCollectionReaderForCompact reader(queryImpl, nssEsc, escStats); - - // get the upper bound index 'pos' using binary search - // get the lower bound index 'ipos' from the null doc, if it exists, otherwise 1 - ESCPreCompactState state; - - auto alpha = ESCCollection::emuBinary(reader, tagToken, valueToken); - if (alpha.has_value() && alpha.value() == 0) { - // No null doc & no entries found for this field/value pair so nothing to compact. - // This can happen if the tag and value tokens were derived from a bogus ECOC - // document, or from an ECOC document decrypted with bogus compaction tokens. - // Skip inserting the compaction placeholder. - return state; - } else if (!alpha.has_value()) { - // only the null doc exists - auto block = ESCCollection::generateId(tagToken, boost::none); - auto r_esc = reader.getById(block); - uassert(6346802, "ESC null document not found", !r_esc.isEmpty()); - - auto nullDoc = uassertStatusOK(ESCCollection::decryptNullDocument(valueToken, r_esc)); - - // +2 to skip over index of placeholder doc from previous compaction - state.pos = nullDoc.position + 2; - state.ipos = state.pos; - state.count = nullDoc.count; - } else { - // one or more entries exist for this field/value pair - auto block = ESCCollection::generateId(tagToken, alpha); - auto r_esc = reader.getById(block); - uassert(6346803, "ESC document not found", !r_esc.isEmpty()); - - auto escDoc = uassertStatusOK(ESCCollection::decryptDocument(valueToken, r_esc)); - - state.pos = alpha.value() + 1; - state.count = escDoc.count; - - // null doc may or may not yet exist - block = ESCCollection::generateId(tagToken, boost::none); - r_esc = reader.getById(block); - if (r_esc.isEmpty()) { - state.ipos = 1; - } else { - auto nullDoc = uassertStatusOK(ESCCollection::decryptNullDocument(valueToken, r_esc)); - state.ipos = nullDoc.position + 2; - } - } - - uassert(6346804, "Invalid position range for ESC compact", state.ipos <= state.pos); - uassert(6346805, "Invalid counter value for ESC compact", state.count > 0); - - // Insert a placeholder at the next ESC position; this is deleted later in compact. - // This serves to trigger a write conflict if another write transaction is - // committed before the current compact transaction commits - if (MONGO_unlikely(fleCompactHangBeforeESCPlaceholderInsert.shouldFail())) { - LOGV2(6548301, "Hanging due to fleCompactHangBeforeESCPlaceholderInsert fail point"); - fleCompactHangBeforeESCPlaceholderInsert.pauseWhileSet(); - } - - auto placeholder = ESCCollection::generateCompactionPlaceholderDocument( - tagToken, valueToken, state.pos, state.count); - StmtId stmtId = kUninitializedStmtId; - auto insertReply = - uassertStatusOK(queryImpl->insertDocuments(nssEsc, {placeholder}, &stmtId, true)); - checkWriteErrors(insertReply); - stats.addInserts(1); - - if (MONGO_unlikely(fleCompactHangAfterESCPlaceholderInsert.shouldFail())) { - LOGV2(6548302, "Hanging due to fleCompactHangAfterESCPlaceholderInsert fail point"); - fleCompactHangAfterESCPlaceholderInsert.pauseWhileSet(); - } - return state; -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Result of preparing the ECC collection for a single field/value pair - * before compaction. - */ -struct ECCPreCompactState { - // total deletions of this field/value pair from EDC - uint64_t count{0}; - // position of the lowest entry - uint64_t ipos{0}; - // position of the highest entry - uint64_t pos{0}; - // result of merging all ECC entries for this field/value pair - std::vector<ECCDocument> g_prime; - // whether the merge reduced the number of ECC entries - bool merged{false}; -}; - -// TODO: SERVER-73303 delete when v2 is enabled by default -ECCPreCompactState prepareECCForCompaction(FLEQueryInterface* queryImpl, - const NamespaceString& nssEcc, - const ECCTwiceDerivedTagToken& tagToken, - const ECCTwiceDerivedValueToken& valueToken, - ECStats* eccStats) { - CompactStatsCounter<ECStats> stats(eccStats); - - TxnCollectionReaderForCompact reader(queryImpl, nssEcc, eccStats); - - ECCPreCompactState state; - bool flag = true; - std::vector<ECCDocument> g; - - // find the null doc - auto block = ECCCollection::generateId(tagToken, boost::none); - auto r_ecc = reader.getById(block); - if (r_ecc.isEmpty()) { - state.pos = 1; - } else { - auto nullDoc = uassertStatusOK(ECCCollection::decryptNullDocument(valueToken, r_ecc)); - state.pos = nullDoc.position + 2; - } - - // get all documents starting from ipos; set pos to one after position of last document found - state.ipos = state.pos; - while (flag) { - block = ECCCollection::generateId(tagToken, state.pos); - r_ecc = reader.getById(block); - if (!r_ecc.isEmpty()) { - auto doc = uassertStatusOK(ECCCollection::decryptDocument(valueToken, r_ecc)); - g.push_back(std::move(doc)); - state.pos += 1; - } else { - flag = false; - } - } - - if (g.empty()) { - // if there are no entries, there must not be a null doc and ipos must be 1 - uassert(6346901, "Found ECC null doc, but no ECC entries", state.ipos == 1); - - // no null doc & no entries found, so nothing to compact - state.pos = 0; - state.ipos = 0; - state.count = 0; - return state; - } - - // merge 'g' - state.g_prime = CompactionHelpers::mergeECCDocuments(g); - dassert(std::is_sorted(g.begin(), g.end())); - dassert(std::is_sorted(state.g_prime.begin(), state.g_prime.end())); - state.merged = (state.g_prime != g); - state.count = CompactionHelpers::countDeleted(state.g_prime); - - if (state.merged) { - // Insert a placeholder at the next ECC position; this is deleted later in compact. - // This serves to trigger a write conflict if another write transaction is - // committed before the current compact transaction commits - auto placeholder = - ECCCollection::generateCompactionDocument(tagToken, valueToken, state.pos); - StmtId stmtId = kUninitializedStmtId; - - if (MONGO_unlikely(fleCompactHangBeforeECCPlaceholderInsert.shouldFail())) { - LOGV2(6548303, "Hanging due to fleCompactHangBeforeECCPlaceholderInsert fail point"); - fleCompactHangBeforeECCPlaceholderInsert.pauseWhileSet(); - } - auto insertReply = - uassertStatusOK(queryImpl->insertDocuments(nssEcc, {placeholder}, &stmtId, true)); - checkWriteErrors(insertReply); - stats.addInserts(1); - - if (MONGO_unlikely(fleCompactHangAfterECCPlaceholderInsert.shouldFail())) { - LOGV2(6548304, "Hanging due to fleCompactHangAfterECCPlaceholderInsert fail point"); - fleCompactHangAfterECCPlaceholderInsert.pauseWhileSet(); - } - } else { - // adjust pos back to the last document found - state.pos -= 1; - } - - return state; -} - } // namespace @@ -457,12 +158,6 @@ EncryptedStateCollectionsNamespaces::createFromDataCollection(const Collection& namespaces.escNss = NamespaceString(db, cfg.getEscCollection().value_or_eval([&f]() { return f("state"_sd); })); - // TODO SERVER-73303 remove when feature flag is enabled. - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - namespaces.eccNss = NamespaceString( - db, cfg.getEccCollection().value_or_eval([&f]() { return f("cache"_sd); })); - } - namespaces.ecocNss = NamespaceString( db, cfg.getEcocCollection().value_or_eval([&f]() { return f("compaction"_sd); })); @@ -484,38 +179,6 @@ EncryptedStateCollectionsNamespaces::createFromDataCollection(const Collection& * that have been encrypted with that token. All entries are returned * in a set in their decrypted form. */ -stdx::unordered_set<ECOCCompactionDocument> getUniqueCompactionDocuments( - FLEQueryInterface* queryImpl, - const CompactStructuredEncryptionData& request, - const NamespaceString& ecocNss, - ECOCStats* ecocStats) { - - CompactStatsCounter<ECOCStats> stats(ecocStats); - - // Initialize a set 'C' and for each compaction token, find all entries - // in ECOC with matching field name. Decrypt entries and add to set 'C'. - stdx::unordered_set<ECOCCompactionDocument> c; - auto compactionTokens = CompactionHelpers::parseCompactionTokens(request.getCompactionTokens()); - - for (auto& compactionToken : compactionTokens) { - auto docs = queryImpl->findDocuments( - ecocNss, BSON(EcocDocument::kFieldNameFieldName << compactionToken.fieldPathName)); - stats.addReads(docs.size()); - - for (auto& doc : docs) { - auto ecocDoc = ECOCCollection::parseAndDecrypt(doc, compactionToken.token); - c.insert(std::move(ecocDoc)); - } - } - return c; -} - -/** - * Parses the compaction tokens from the compact request, and - * for each one, retrieves the unique entries in the ECOC collection - * that have been encrypted with that token. All entries are returned - * in a set in their decrypted form. - */ stdx::unordered_set<ECOCCompactionDocumentV2> getUniqueCompactionDocumentsV2( FLEQueryInterface* queryImpl, const CompactStructuredEncryptionData& request, @@ -542,80 +205,6 @@ stdx::unordered_set<ECOCCompactionDocumentV2> getUniqueCompactionDocumentsV2( return c; } -void compactOneFieldValuePair(FLEQueryInterface* queryImpl, - const ECOCCompactionDocument& ecocDoc, - const EncryptedStateCollectionsNamespaces& namespaces, - ECStats* escStats, - ECStats* eccStats) { - // PART 1 - // prepare the ESC, and get back the highest counter value before the placeholder - // document, ipos, and pos - auto escTagToken = FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedTagToken(ecocDoc.esc); - auto escValueToken = - FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedValueToken(ecocDoc.esc); - auto escState = - prepareESCForCompaction(queryImpl, namespaces.escNss, escTagToken, escValueToken, escStats); - - // PART 2 - // prepare the ECC, and get back the merged set 'g_prime', whether (g_prime != g), - // ipos_prime, and pos_prime - auto eccTagToken = FLETwiceDerivedTokenGenerator::generateECCTwiceDerivedTagToken(ecocDoc.ecc); - auto eccValueToken = - FLETwiceDerivedTokenGenerator::generateECCTwiceDerivedValueToken(ecocDoc.ecc); - auto eccState = - prepareECCForCompaction(queryImpl, namespaces.eccNss, eccTagToken, eccValueToken, eccStats); - - // PART 3 - // A. compact the ECC - StmtId stmtId = kUninitializedStmtId; - - if (eccState.count != 0) { - if (eccState.merged) { - CompactStatsCounter<ECStats> stats(eccStats); - - // a. for each entry in g_prime at index k, insert - // {_id: F(eccTagToken, pos'+ k), value: Enc(eccValueToken, g_prime[k])} - for (auto k = eccState.g_prime.size(); k > 0; k--) { - const auto& range = eccState.g_prime[k - 1]; - auto insertReply = uassertStatusOK(queryImpl->insertDocuments( - namespaces.eccNss, - {ECCCollection::generateDocument( - eccTagToken, eccValueToken, eccState.pos + k, range.start, range.end)}, - &stmtId, - true)); - checkWriteErrors(insertReply); - stats.addInserts(1); - } - - // b & c. update or insert the ECC null doc - bool hasNullDoc = (eccState.ipos > 1); - auto newNullDoc = - ECCCollection::generateNullDocument(eccTagToken, eccValueToken, eccState.pos - 1); - upsertNullDocument(queryImpl, hasNullDoc, newNullDoc, namespaces.eccNss, eccStats); - - // d. delete entries between ipos' and pos', inclusive - for (auto k = eccState.ipos; k <= eccState.pos; k++) { - deleteECCDocument(queryImpl, namespaces.eccNss, k, eccTagToken, eccStats); - } - } - } - - // B. compact the ESC - if (escState.count != 0) { - // Delete ESC entries between ipos and pos, inclusive. - // The compaction placeholder is at index pos, so it will be deleted as well. - for (auto k = escState.ipos; k <= escState.pos; k++) { - deleteESCDocument(queryImpl, namespaces.escNss, k, escTagToken, escStats); - } - - // update or insert the ESC null doc - bool hasNullDoc = (escState.ipos > 1); - auto newNullDoc = ESCCollection::generateNullDocument( - escTagToken, escValueToken, escState.pos - 1, escState.count); - upsertNullDocument(queryImpl, hasNullDoc, newNullDoc, namespaces.escNss, escStats); - } -} - void compactOneFieldValuePairV2(FLEQueryInterface* queryImpl, const ECOCCompactionDocumentV2& ecocDoc, const NamespaceString& escNss, @@ -674,82 +263,6 @@ void compactOneFieldValuePairV2(FLEQueryInterface* queryImpl, stats.addInserts(1); } -CompactStats processFLECompact(OperationContext* opCtx, - const CompactStructuredEncryptionData& request, - GetTxnCallback getTxn, - const EncryptedStateCollectionsNamespaces& namespaces) { - auto ecocStats = std::make_shared<ECOCStats>(); - auto escStats = std::make_shared<ECStats>(); - auto eccStats = std::make_shared<ECStats>(); - auto c = std::make_shared<stdx::unordered_set<ECOCCompactionDocument>>(); - - if (MONGO_unlikely(fleCompactFailBeforeECOCRead.shouldFail())) { - uasserted(6599901, "Failed compact due to fleCompactFailBeforeECOCRead fail point"); - } - - // Read the ECOC documents in a transaction - { - std::shared_ptr<txn_api::SyncTransactionWithRetries> trun = getTxn(opCtx); - - // The function that handles the transaction may outlive this function so we need to use - // shared_ptrs - auto argsBlock = std::make_tuple(request, namespaces); - auto sharedBlock = std::make_shared<decltype(argsBlock)>(argsBlock); - - auto swResult = trun->runNoThrow( - opCtx, - [sharedBlock, c, ecocStats](const txn_api::TransactionClient& txnClient, - ExecutorPtr txnExec) { - FLEQueryInterfaceImpl queryImpl(txnClient, getGlobalServiceContext()); - - auto [request2, namespaces2] = *sharedBlock.get(); - - *c = getUniqueCompactionDocuments( - &queryImpl, request2, namespaces2.ecocRenameNss, ecocStats.get()); - - return SemiFuture<void>::makeReady(); - }); - - uassertStatusOK(swResult); - uassertStatusOK(swResult.getValue().getEffectiveStatus()); - } - - // Each entry in 'C' represents a unique field/value pair. For each field/value pair, - // compact the ESC & ECC entries for that field/value pair in one transaction. - for (auto& ecocDoc : *c) { - // start a new transaction - std::shared_ptr<txn_api::SyncTransactionWithRetries> trun = getTxn(opCtx); - - // The function that handles the transaction may outlive this function so we need to use - // shared_ptrs - auto argsBlock = std::make_tuple(ecocDoc, namespaces); - auto sharedBlock = std::make_shared<decltype(argsBlock)>(argsBlock); - - auto swResult = trun->runNoThrow( - opCtx, - [sharedBlock, escStats, eccStats](const txn_api::TransactionClient& txnClient, - ExecutorPtr txnExec) { - FLEQueryInterfaceImpl queryImpl(txnClient, getGlobalServiceContext()); - - auto [ecocDoc2, namespaces2] = *sharedBlock.get(); - - compactOneFieldValuePair( - &queryImpl, ecocDoc2, namespaces2, escStats.get(), eccStats.get()); - - return SemiFuture<void>::makeReady(); - }); - - uassertStatusOK(swResult); - uassertStatusOK(swResult.getValue().getEffectiveStatus()); - } - - CompactStats stats(*ecocStats, *escStats); - stats.setEcc(*eccStats); - FLEStatusSection::get().updateCompactionStats(stats); - - return stats; -} - void processFLECompactV2(OperationContext* opCtx, const CompactStructuredEncryptionData& request, GetTxnCallback getTxn, diff --git a/src/mongo/db/commands/fle2_compact.h b/src/mongo/db/commands/fle2_compact.h index ff351703573..2af0e930f3a 100644 --- a/src/mongo/db/commands/fle2_compact.h +++ b/src/mongo/db/commands/fle2_compact.h @@ -44,8 +44,6 @@ struct EncryptedStateCollectionsNamespaces { NamespaceString escNss; NamespaceString ecocNss; NamespaceString ecocRenameNss; - // TODO: SERVER-73303 delete when v2 is enabled by default - NamespaceString eccNss; }; /** @@ -53,12 +51,6 @@ struct EncryptedStateCollectionsNamespaces { */ void validateCompactRequest(const CompactStructuredEncryptionData& request, const Collection& edc); -// TODO: SERVER-73303 delete when v2 is enabled by default -CompactStats processFLECompact(OperationContext* opCtx, - const CompactStructuredEncryptionData& request, - GetTxnCallback getTxn, - const EncryptedStateCollectionsNamespaces& namespaces); - void processFLECompactV2(OperationContext* opCtx, const CompactStructuredEncryptionData& request, GetTxnCallback getTxn, @@ -66,18 +58,6 @@ void processFLECompactV2(OperationContext* opCtx, ECStats* escStats, ECOCStats* ecocStats); -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Get all unique documents in the ECOC collection in their decrypted form. - * - * Used by unit tests. - */ -stdx::unordered_set<ECOCCompactionDocument> getUniqueCompactionDocuments( - FLEQueryInterface* queryImpl, - const CompactStructuredEncryptionData& request, - const NamespaceString& ecocNss, - ECOCStats* ecocStats); - /** * Get all unique documents in the ECOC collection in their decrypted form. * @@ -90,19 +70,6 @@ stdx::unordered_set<ECOCCompactionDocumentV2> getUniqueCompactionDocumentsV2( ECOCStats* ecocStats); -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Performs compaction of the ESC and ECC entries for the encrypted field/value pair - * whose tokens are in the provided ECOC compaction document. - * - * Used by unit tests. - */ -void compactOneFieldValuePair(FLEQueryInterface* queryImpl, - const ECOCCompactionDocument& ecocDoc, - const EncryptedStateCollectionsNamespaces& namespaces, - ECStats* escStats, - ECStats* eccStats); - /** * Performs compaction of the ESC entries for the encrypted field/value pair * whose tokens are in the provided ECOC compaction document. diff --git a/src/mongo/db/commands/fle2_compact_cmd.cpp b/src/mongo/db/commands/fle2_compact_cmd.cpp index 5fcbe58d727..e0fa4440f22 100644 --- a/src/mongo/db/commands/fle2_compact_cmd.cpp +++ b/src/mongo/db/commands/fle2_compact_cmd.cpp @@ -75,8 +75,6 @@ CompactStats compactEncryptedCompactionCollection(OperationContext* opCtx, Lock::ExclusiveLock fleCompactCommandLock(opCtx, commandMutex); const auto& edcNss = request.getNamespace(); - const bool useV1Protocol = - !gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility); LOGV2(6319900, "Compacting the encrypted compaction collection", logAttrs(edcNss)); @@ -110,24 +108,10 @@ CompactStats compactEncryptedCompactionCollection(OperationContext* opCtx, CompactStats stats({}, {}); FLECompactESCDeleteSet escDeleteSet; - if (useV1Protocol) { - stats.setEcc({}); - } - if (!ecoc && !ecocRename) { // nothing to compact LOGV2(6548306, "Skipping compaction as there is no ECOC collection to compact"); return stats; - } else if (ecoc && !ecocRename && useV1Protocol) { - // TODO: SERVER-73303 delete this branch when v2 is enabled by default - LOGV2(6319901, - "Renaming the encrypted compaction collection", - "ecocNss"_attr = namespaces.ecocNss, - "ecocRenameNss"_attr = namespaces.ecocRenameNss); - RenameCollectionOptions renameOpts; - validateAndRunRenameCollection( - opCtx, namespaces.ecocNss, namespaces.ecocRenameNss, renameOpts); - ecoc = nullptr; } else if (ecoc && !ecocRename) { // load the random set of ESC non-anchor entries to be deleted post-compact auto memoryLimit = @@ -188,30 +172,21 @@ CompactStats compactEncryptedCompactionCollection(OperationContext* opCtx, << " no longer exists prior to compaction", tempEcocColl.getCollection()); - if (useV1Protocol) { - // TODO: SERVER-73303 delete this branch when v2 is enabled by default - stats = - processFLECompact(opCtx, request, &getTransactionWithRetriesForMongoD, namespaces); - } else { - processFLECompactV2(opCtx, - request, - &getTransactionWithRetriesForMongoD, - namespaces, - &stats.getEsc(), - &stats.getEcoc()); - } + processFLECompactV2(opCtx, + request, + &getTransactionWithRetriesForMongoD, + namespaces, + &stats.getEsc(), + &stats.getEcoc()); } - if (!useV1Protocol) { - auto tagsPerDelete = - ServerParameterSet::getClusterParameterSet() - ->get<ClusterParameterWithStorage<FLECompactionOptions>>("fleCompactionOptions") - ->getValue(boost::none) - .getMaxESCEntriesPerCompactionDelete(); - cleanupESCNonAnchors( - opCtx, namespaces.escNss, escDeleteSet, tagsPerDelete, &stats.getEsc()); - FLEStatusSection::get().updateCompactionStats(stats); - } + auto tagsPerDelete = + ServerParameterSet::getClusterParameterSet() + ->get<ClusterParameterWithStorage<FLECompactionOptions>>("fleCompactionOptions") + ->getValue(boost::none) + .getMaxESCEntriesPerCompactionDelete(); + cleanupESCNonAnchors(opCtx, namespaces.escNss, escDeleteSet, tagsPerDelete, &stats.getEsc()); + FLEStatusSection::get().updateCompactionStats(stats); if (MONGO_unlikely(fleCompactSkipECOCDropUnsharded.shouldFail())) { LOGV2(7299612, diff --git a/src/mongo/db/commands/fle2_get_count_info_command.cpp b/src/mongo/db/commands/fle2_get_count_info_command.cpp index 0ad2d793e19..fe5b82fe9b3 100644 --- a/src/mongo/db/commands/fle2_get_count_info_command.cpp +++ b/src/mongo/db/commands/fle2_get_count_info_command.cpp @@ -103,10 +103,6 @@ QECountInfosReply getTagsLocal(OperationContext* opCtx, CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; - uassert(741503, - "FeatureFlagFLE2ProtocolVersion2 is not enabled", - gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)); - auto nestedTokens = toNestedTokens(request.getTokens()); auto countInfoSets = diff --git a/src/mongo/db/commands/fle_compact_test.cpp b/src/mongo/db/commands/fle_compact_test.cpp index 7eeba42ea14..da83ffe3436 100644 --- a/src/mongo/db/commands/fle_compact_test.cpp +++ b/src/mongo/db/commands/fle_compact_test.cpp @@ -185,10 +185,8 @@ protected: std::vector<char> generatePlaceholder(UUID keyId, BSONElement value); void doSingleInsert(int id, BSONObj encryptedFieldsObj); - void doSingleDelete(int id, BSONObj encryptedFieldsObj); void insertFieldValues(StringData fieldName, std::map<std::string, InsertionState>& values); - void deleteFieldValues(StringData fieldName, std::map<std::string, InsertionState>& values); protected: ServiceContext::UniqueOperationContext _opCtx; @@ -385,33 +383,6 @@ void FleCompactTest::doSingleInsert(int id, BSONObj encryptedFieldsObj) { _queryImpl.get(), _namespaces.edcNss, serverPayload, efc, &stmtId, result, false)); } -void FleCompactTest::doSingleDelete(int id, BSONObj encryptedFieldsObj) { - auto efc = - generateEncryptedFieldConfig(encryptedFieldsObj.getFieldNames<std::set<std::string>>()); - - auto doc = EncryptionInformationHelpers::encryptionInformationSerializeForDelete( - _namespaces.edcNss, efc, &_keyVault); - - auto ei = EncryptionInformation::parse(IDLParserContext("test"), doc); - - write_ops::DeleteOpEntry entry; - entry.setQ(BSON("_id" << id)); - entry.setMulti(false); - - write_ops::DeleteCommandRequest deleteRequest(_namespaces.edcNss); - deleteRequest.setDeletes({entry}); - deleteRequest.getWriteCommandRequestBase().setEncryptionInformation(ei); - - std::unique_ptr<CollatorInterface> collator; - auto expCtx = make_intrusive<ExpressionContext>(_opCtx.get(), - std::move(collator), - deleteRequest.getNamespace(), - deleteRequest.getLegacyRuntimeConstants(), - deleteRequest.getLet()); - - processDelete(_queryImpl.get(), expCtx, deleteRequest); -} - void FleCompactTest::insertFieldValues(StringData field, std::map<std::string, InsertionState>& values) { static int insertId = 1; @@ -429,32 +400,11 @@ void FleCompactTest::insertFieldValues(StringData field, } } -void FleCompactTest::deleteFieldValues(StringData field, - std::map<std::string, InsertionState>& values) { - for (auto& [value, state] : values) { - BSONObjBuilder builder; - builder.append(field, value); - auto entry = builder.obj(); - - for (auto& range : state.toDeleteRanges) { - for (auto ctr = range.first; ctr <= range.second; ctr++) { - if (state.insertedIds.find(ctr) == state.insertedIds.end()) { - continue; - } - doSingleDelete(state.insertedIds[ctr], entry); - state.insertedIds.erase(ctr); - } - } - } -} TEST_F(FleCompactTest, GetUniqueECOCDocsFromEmptyECOC) { ECOCStats stats; std::set<std::string> fieldSet = {"first", "ssn"}; auto cmd = generateCompactCommand(fieldSet); - auto docsV1 = getUniqueCompactionDocuments(_queryImpl.get(), cmd, _namespaces.ecocNss, &stats); - ASSERT(docsV1.empty()); - auto docs = getUniqueCompactionDocumentsV2(_queryImpl.get(), cmd, _namespaces.ecocNss, &stats); ASSERT(docs.empty()); } @@ -505,7 +455,6 @@ TEST_F(FleCompactTest, CompactValueV2_NoNonAnchors) { } TEST_F(FleCompactTest, CompactValueV2_NoNullAnchors) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); ECStats escStats; std::map<std::string, InsertionState> values; constexpr auto key = "first"_sd; @@ -548,7 +497,6 @@ TEST_F(FleCompactTest, CompactValueV2_NoNullAnchors) { } TEST_F(FleCompactTest, RandomESCNonAnchorDeletions) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); ECStats escStats; constexpr auto key = "first"_sd; const std::string value = "roger"; diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index 2d9bc0d74b0..31a0757368d 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -346,7 +346,7 @@ public: // TODO SERVER-68373: Remove once 7.0 becomes last LTS if (serverGlobalParams.clusterRole.has(ClusterRole::ShardServer) && - gFeatureFlagFLE2ProtocolVersion2.isEnabledOnVersion(requestedVersion)) { + mongo::gFeatureFlagFLE2CompactForProtocolV2.isEnabledOnVersion(requestedVersion)) { ShardingDDLCoordinatorService::getService(opCtx) ->waitForCoordinatorsOfGivenTypeToComplete( opCtx, @@ -559,8 +559,8 @@ public: // TODO SERVER-68373: Remove once 7.0 becomes last LTS if (serverGlobalParams.clusterRole.has(ClusterRole::ShardServer) && requestedVersion > actualVersion && - gFeatureFlagFLE2ProtocolVersion2.isEnabledOnTargetFCVButDisabledOnOriginalFCV( - requestedVersion, actualVersion)) { + mongo::gFeatureFlagFLE2CompactForProtocolV2 + .isEnabledOnTargetFCVButDisabledOnOriginalFCV(requestedVersion, actualVersion)) { ShardingDDLCoordinatorService::getService(opCtx) ->waitForCoordinatorsOfGivenTypeToComplete( opCtx, DDLCoordinatorTypeEnum::kCompactStructuredEncryptionDataPre70Compatible); @@ -639,8 +639,8 @@ private: // TODO SERVER-68373 remove once 7.0 becomes last LTS if (isDowngrading && - gFeatureFlagFLE2ProtocolVersion2.isDisabledOnTargetFCVButEnabledOnOriginalFCV( - requestedVersion, originalVersion)) { + mongo::gFeatureFlagFLE2CompactForProtocolV2 + .isDisabledOnTargetFCVButEnabledOnOriginalFCV(requestedVersion, originalVersion)) { // Drain the QE compact coordinator because it persists state that is // not backwards compatible with earlier versions. ShardingDDLCoordinatorService::getService(opCtx) diff --git a/src/mongo/db/fle_crud.cpp b/src/mongo/db/fle_crud.cpp index 974605c8df9..ed6af4024aa 100644 --- a/src/mongo/db/fle_crud.cpp +++ b/src/mongo/db/fle_crud.cpp @@ -448,15 +448,6 @@ write_ops::DeleteCommandReply processDelete(OperationContext* opCtx, { auto deletes = deleteRequest.getDeletes(); uassert(6371302, "Only single document deletes are permitted", deletes.size() == 1); - - // TODO: SERVER-73303 delete when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - auto deleteOpEntry = deletes[0]; - - uassert(6371303, - "FLE only supports single document deletes", - deleteOpEntry.getMulti() == false); - } } CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; @@ -625,104 +616,6 @@ write_ops::UpdateCommandReply processUpdate(OperationContext* opCtx, namespace { -// TODO: SERVER-73303 delete when v2 is enabled by default -void processFieldsForInsertV1(FLEQueryInterface* queryImpl, - const NamespaceString& edcNss, - std::vector<EDCServerPayloadInfo>& serverPayload, - const EncryptedFieldConfig& efc, - int32_t* pStmtId, - bool bypassDocumentValidation) { - - const NamespaceString nssEsc = NamespaceStringUtil::parseNamespaceFromRequest( - edcNss.dbName(), efc.getEscCollection().value()); - - auto docCount = queryImpl->countDocuments(nssEsc); - - TxnCollectionReader reader(docCount, queryImpl, nssEsc); - - for (auto& payload : serverPayload) { - - const auto insertTokens = [&](ConstDataRange encryptedTokens, - ConstDataRange escDerivedToken) { - int position = 1; - int count = 1; - - auto escToken = EDCServerPayloadInfo::getESCToken(escDerivedToken); - auto tagToken = - FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedTagToken(escToken); - auto valueToken = - FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedValueToken(escToken); - - auto alpha = ESCCollection::emuBinary(reader, tagToken, valueToken); - - if (alpha.has_value() && alpha.value() == 0) { - position = 1; - count = 1; - } else if (!alpha.has_value()) { - auto block = ESCCollection::generateId(tagToken, boost::none); - - auto r_esc = reader.getById(block); - uassert(6371203, "ESC document not found", !r_esc.isEmpty()); - - auto escNullDoc = - uassertStatusOK(ESCCollection::decryptNullDocument(valueToken, r_esc)); - - position = escNullDoc.position + 2; - count = escNullDoc.count + 1; - } else { - auto block = ESCCollection::generateId(tagToken, alpha); - - auto r_esc = reader.getById(block); - uassert(6371204, "ESC document not found", !r_esc.isEmpty()); - - auto escDoc = uassertStatusOK(ESCCollection::decryptDocument(valueToken, r_esc)); - - position = alpha.value() + 1; - count = escDoc.count + 1; - - if (escDoc.compactionPlaceholder) { - uassertStatusOK(Status(ErrorCodes::FLECompactionPlaceholder, - "Found ESC contention placeholder")); - } - } - - payload.counts.push_back(count); - - auto escInsertReply = uassertStatusOK(queryImpl->insertDocuments( - nssEsc, - {ESCCollection::generateInsertDocument(tagToken, valueToken, position, count)}, - pStmtId, - true)); - checkWriteErrors(escInsertReply); - - - const NamespaceString nssEcoc = NamespaceStringUtil::parseNamespaceFromRequest( - edcNss.dbName(), efc.getEcocCollection().value()); - - // TODO - should we make this a batch of ECOC updates? - const auto ecocInsertReply = uassertStatusOK(queryImpl->insertDocuments( - nssEcoc, - {ECOCCollection::generateDocument(payload.fieldPathName, encryptedTokens)}, - pStmtId, - false, - bypassDocumentValidation)); - checkWriteErrors(ecocInsertReply); - }; - - payload.counts.clear(); - const bool isRangePayload = payload.payload.getEdgeTokenSet().has_value(); - if (isRangePayload) { - const auto ets = payload.payload.getEdgeTokenSet().get(); - for (size_t i = 0; i < ets.size(); ++i) { - insertTokens(ets[i].getEncryptedTokens(), ets[i].getEscDerivedToken()); - } - } else { - insertTokens(payload.payload.getEncryptedTokens(), - payload.payload.getEscDerivedToken()); - } - } -} - void processFieldsForInsertV2(FLEQueryInterface* queryImpl, const NamespaceString& edcNss, std::vector<EDCServerPayloadInfo>& serverPayload, @@ -827,149 +720,8 @@ void processFieldsForInsert(FLEQueryInterface* queryImpl, const EncryptedFieldConfig& efc, int32_t* pStmtId, bool bypassDocumentValidation) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - processFieldsForInsertV2( - queryImpl, edcNss, serverPayload, efc, pStmtId, bypassDocumentValidation); - } else { - processFieldsForInsertV1( - queryImpl, edcNss, serverPayload, efc, pStmtId, bypassDocumentValidation); - } -} - -void processRemovedFieldsHelper(FLEQueryInterface* queryImpl, - const EncryptedFieldConfig& efc, - const ESCDerivedFromDataTokenAndContentionFactorToken& esc, - const ECCDerivedFromDataTokenAndContentionFactorToken& ecc, - uint64_t count, - const EDCIndexedFields& deletedField, - const FLEDeleteToken& deleteToken, - const TxnCollectionReader& reader, - const NamespaceString& eccNss, - const NamespaceString& edcNss, - int32_t* pStmtId) { - auto tagToken = FLETwiceDerivedTokenGenerator::generateECCTwiceDerivedTagToken(ecc); - auto valueToken = FLETwiceDerivedTokenGenerator::generateECCTwiceDerivedValueToken(ecc); - - auto alpha = ECCCollection::emuBinary(reader, tagToken, valueToken); - - uint64_t index = 0; - if (alpha.has_value() && alpha.value() == 0) { - index = 1; - } else if (!alpha.has_value()) { - auto block = ECCCollection::generateId(tagToken, boost::none); - - auto r_ecc = reader.getById(block); - uassert(6371306, "ECC null document not found", !r_ecc.isEmpty()); - - auto eccNullDoc = uassertStatusOK(ECCCollection::decryptNullDocument(valueToken, r_ecc)); - index = eccNullDoc.position + 2; - } else { - auto block = ECCCollection::generateId(tagToken, alpha); - - auto r_ecc = reader.getById(block); - uassert(6371307, "ECC document not found", !r_ecc.isEmpty()); - - auto eccDoc = uassertStatusOK(ECCCollection::decryptDocument(valueToken, r_ecc)); - - if (eccDoc.valueType == ECCValueType::kCompactionPlaceholder) { - uassertStatusOK( - Status(ErrorCodes::FLECompactionPlaceholder, "Found contention placeholder")); - } - - index = alpha.value() + 1; - } - - auto eccInsertReply = uassertStatusOK(queryImpl->insertDocuments( - eccNss, - {ECCCollection::generateDocument(tagToken, valueToken, index, count)}, - pStmtId, - true)); - checkWriteErrors(eccInsertReply); - - const NamespaceString nssEcoc = NamespaceStringUtil::parseNamespaceFromRequest( - edcNss.dbName(), efc.getEcocCollection().value()); - - // TODO - make this a batch of ECOC updates? - EncryptedStateCollectionTokens tokens(esc, ecc); - auto encryptedTokens = uassertStatusOK(tokens.serialize(deleteToken.ecocToken)); - auto ecocInsertReply = uassertStatusOK(queryImpl->insertDocuments( - nssEcoc, - {ECOCCollection::generateDocument(deletedField.fieldPathName, encryptedTokens)}, - pStmtId, - false)); - checkWriteErrors(ecocInsertReply); -}; - -void processRemovedFields(FLEQueryInterface* queryImpl, - const NamespaceString& edcNss, - const EncryptedFieldConfig& efc, - const StringMap<FLEDeleteToken>& tokenMap, - const std::vector<EDCIndexedFields>& deletedFields, - int32_t* pStmtId) { - - const NamespaceString eccNss = NamespaceStringUtil::parseNamespaceFromRequest( - edcNss.dbName(), efc.getEccCollection().value()); - - auto docCount = queryImpl->countDocuments(eccNss); - - TxnCollectionReader reader(docCount, queryImpl, eccNss); - - for (const auto& deletedField : deletedFields) { - // TODO - verify each indexed fields is listed in EncryptionInformation for the - // schema - - auto it = tokenMap.find(deletedField.fieldPathName); - uassert(6371304, - str::stream() << "Could not find delete token for field: " - << deletedField.fieldPathName, - it != tokenMap.end()); - - auto deleteToken = it->second; - - auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(deletedField.value); - - // TODO - add support other types - uassert(6371305, - "Ony support deleting equality indexed fields or range indexed fields", - encryptedTypeBinding == EncryptedBinDataType::kFLE2EqualityIndexedValue || - encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValue); - - if (encryptedTypeBinding == EncryptedBinDataType::kFLE2EqualityIndexedValue) { - auto plainTextField = - uassertStatusOK(FLE2IndexedEqualityEncryptedValue::decryptAndParse( - deleteToken.serverEncryptionToken, subCdr)); - processRemovedFieldsHelper(queryImpl, - efc, - plainTextField.esc, - plainTextField.ecc, - plainTextField.count, - deletedField, - deleteToken, - reader, - eccNss, - edcNss, - pStmtId); - } else { - auto plainTextField = uassertStatusOK(FLE2IndexedRangeEncryptedValue::decryptAndParse( - deleteToken.serverEncryptionToken, subCdr)); - - for (size_t i = 0; i < plainTextField.counters.size(); i++) { - const auto& edgeTokenSet = plainTextField.tokens[i]; - const auto& count = plainTextField.counters[i]; - processRemovedFieldsHelper(queryImpl, - efc, - edgeTokenSet.esc, - edgeTokenSet.ecc, - count, - deletedField, - deleteToken, - reader, - eccNss, - edcNss, - pStmtId); - } - } - } + processFieldsForInsertV2( + queryImpl, edcNss, serverPayload, efc, pStmtId, bypassDocumentValidation); } template <typename ReplyType> @@ -1137,11 +889,7 @@ write_ops::DeleteCommandReply processDelete(FLEQueryInterface* queryImpl, auto efc = EncryptionInformationHelpers::getAndValidateSchema(edcNss, ei); - // TODO: SERVER-73303 delete when v2 is enabled by default - StringMap<FLEDeleteToken> tokenMap; - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - tokenMap = EncryptionInformationHelpers::getDeleteTokens(edcNss, ei); - } else if (ei.getDeleteTokens().has_value()) { + if (ei.getDeleteTokens().has_value()) { uasserted(7293102, "Illegal delete tokens encountered in EncryptionInformation"); } @@ -1157,27 +905,6 @@ write_ops::DeleteCommandReply processDelete(FLEQueryInterface* queryImpl, newDeleteRequest.getWriteCommandRequestBase().setStmtIds(boost::none); - // TODO: SERVER-73303 delete when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - newDeleteRequest.getWriteCommandRequestBase().setStmtId(stmtId); - ++stmtId; - - auto [deleteReply, deletedDocument] = - queryImpl->deleteWithPreimage(edcNss, ei, newDeleteRequest); - checkWriteErrors(deleteReply); - - // If the delete did not actually delete anything, we are done - if (deletedDocument.isEmpty()) { - write_ops::DeleteCommandReply reply; - reply.getWriteCommandReplyBase().setN(0); - return reply; - } - auto deletedFields = EDCServerCollection::getEncryptedIndexedFields(deletedDocument); - - processRemovedFields(queryImpl, edcNss, efc, tokenMap, deletedFields, &stmtId); - return deleteReply; - } - newDeleteRequest.getWriteCommandRequestBase().setStmtId(boost::none); newDeleteRequest.getWriteCommandRequestBase().setEncryptionInformation(boost::none); auto deleteReply = queryImpl->deleteDocument(edcNss, stmtId, newDeleteRequest); @@ -1224,11 +951,7 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, auto efc = EncryptionInformationHelpers::getAndValidateSchema(edcNss, ei); - // TODO: SERVER-73303 delete when v2 is enabled by default - StringMap<FLEDeleteToken> tokenMap; - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - tokenMap = EncryptionInformationHelpers::getDeleteTokens(edcNss, ei); - } else if (ei.getDeleteTokens().has_value()) { + if (ei.getDeleteTokens().has_value()) { uasserted(7293201, "Illegal delete tokens encountered in EncryptionInformation"); } @@ -1334,30 +1057,6 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, auto originalFields = EDCServerCollection::getEncryptedIndexedFields(originalDocument); auto newFields = EDCServerCollection::getEncryptedIndexedFields(newDocument); - // TODO: SERVER-73303 delete when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - - auto deletedFields = EDCServerCollection::getRemovedFields(originalFields, newFields); - - processRemovedFields(queryImpl, edcNss, efc, tokenMap, deletedFields, &stmtId); - - // Step 6 ---- - BSONObj pullUpdate = - EDCServerCollection::generateUpdateToRemoveTags(deletedFields, tokenMap); - auto pullUpdateOpEntry = write_ops::UpdateOpEntry(); - pullUpdateOpEntry.setUpsert(false); - pullUpdateOpEntry.setMulti(false); - pullUpdateOpEntry.setQ(BSON("_id"_sd << idElement)); - pullUpdateOpEntry.setU(mongo::write_ops::UpdateModification( - pullUpdate, write_ops::UpdateModification::ModifierUpdateTag{})); - newUpdateRequest.setUpdates({pullUpdateOpEntry}); - newUpdateRequest.getWriteCommandRequestBase().setStmtId(boost::none); - newUpdateRequest.setLegacyRuntimeConstants(boost::none); - newUpdateRequest.getWriteCommandRequestBase().setEncryptionInformation(boost::none); - /* ignore */ queryImpl->update(edcNss, stmtId, newUpdateRequest); - - return updateReply; - } // Step 6 ---- // GarbageCollect steps: @@ -1512,11 +1211,7 @@ write_ops::FindAndModifyCommandReply processFindAndModify( auto efc = EncryptionInformationHelpers::getAndValidateSchema(edcNss, ei); - // TODO: SERVER-73303 delete when v2 is enabled by default - StringMap<FLEDeleteToken> tokenMap; - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - tokenMap = EncryptionInformationHelpers::getDeleteTokens(edcNss, ei); - } else if (ei.getDeleteTokens().has_value()) { + if (ei.getDeleteTokens().has_value()) { uasserted(7293301, "Illegal delete tokens encountered in EncryptionInformation"); } @@ -1605,62 +1300,6 @@ write_ops::FindAndModifyCommandReply processFindAndModify( "Missing _id field in pre-image document, the fields document must contain _id", idElement.fieldNameStringData() == "_id"_sd); - // TODO: SERVER-73303 remove once v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - BSONObj newDocument; - std::vector<EDCIndexedFields> newFields; - - // Is this a delete - bool isDelete = findAndModifyRequest.getRemove().value_or(false); - - // Unlike update, there will not always be a new document since users can delete the - // document - if (!isDelete) { - newDocument = queryImpl->getById(edcNss, idElement); - - // Fail if we could not find the new document - uassert(6371404, "Could not find pre-image document by _id", !newDocument.isEmpty()); - - if (hasIndexedFieldsInSchema(efc.getFields())) { - // Check the user did not remove/destroy the __safeContent__ array. If there are no - // indexed fields, then there will not be a safeContent array in the document. - FLEClientCrypto::validateTagsArray(newDocument); - } - - newFields = EDCServerCollection::getEncryptedIndexedFields(newDocument); - } - - // Step 5 ---- - auto originalFields = EDCServerCollection::getEncryptedIndexedFields(originalDocument); - auto deletedFields = EDCServerCollection::getRemovedFields(originalFields, newFields); - - processRemovedFields(queryImpl, edcNss, efc, tokenMap, deletedFields, &stmtId); - - // Step 6 ---- - // We don't need to make a second update in the case of a delete - if (!isDelete) { - BSONObj pullUpdate = - EDCServerCollection::generateUpdateToRemoveTags(deletedFields, tokenMap); - auto newUpdateRequest = - write_ops::UpdateCommandRequest(findAndModifyRequest.getNamespace()); - auto pullUpdateOpEntry = write_ops::UpdateOpEntry(); - pullUpdateOpEntry.setUpsert(false); - pullUpdateOpEntry.setMulti(false); - pullUpdateOpEntry.setQ(BSON("_id"_sd << idElement)); - pullUpdateOpEntry.setU(mongo::write_ops::UpdateModification( - pullUpdate, write_ops::UpdateModification::ModifierUpdateTag{})); - newUpdateRequest.setUpdates({pullUpdateOpEntry}); - newUpdateRequest.setLegacyRuntimeConstants(boost::none); - newUpdateRequest.getWriteCommandRequestBase().setStmtId(boost::none); - newUpdateRequest.getWriteCommandRequestBase().setEncryptionInformation(boost::none); - - auto finalUpdateReply = queryImpl->update(edcNss, stmtId, newUpdateRequest); - checkWriteErrors(finalUpdateReply); - } - - return reply; - } - // Is this a delete? If so, there's no need to GarbageCollect. if (findAndModifyRequest.getRemove().value_or(false)) { return reply; diff --git a/src/mongo/db/fle_crud_test.cpp b/src/mongo/db/fle_crud_test.cpp index 5c624920d27..65739ebba8d 100644 --- a/src/mongo/db/fle_crud_test.cpp +++ b/src/mongo/db/fle_crud_test.cpp @@ -941,7 +941,7 @@ TEST_F(FleCrudTest, InsertV1PayloadAgainstV2Protocol) { ASSERT_THROWS_CODE( processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, document, false), DBException, - 7291907); + 6379103); } // Test insert of v1 FLEUnindexedEncryptedValue is rejected if v2 is enabled. @@ -992,9 +992,10 @@ TEST_F(FleCrudTest, InsertUnindexedV1AgainstV2Protocol) { ASSERT_THROWS_CODE( processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, document, false), DBException, - 7413902); + 6379103); } + // Insert and delete one document TEST_F(FleCrudTest, InsertAndDeleteOne) { auto doc = BSON("encrypted" @@ -1498,7 +1499,6 @@ TEST_F(FleTagsTest, InsertTwoDifferent) { } TEST_F(FleTagsTest, InsertAndDeleteOne) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto doc = BSON("encrypted" << "a"); @@ -1509,7 +1509,6 @@ TEST_F(FleTagsTest, InsertAndDeleteOne) { } TEST_F(FleTagsTest, InsertTwoSameAndDeleteOne) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto doc = BSON("encrypted" << "a"); @@ -1521,7 +1520,6 @@ TEST_F(FleTagsTest, InsertTwoSameAndDeleteOne) { } TEST_F(FleTagsTest, InsertTwoDifferentAndDeleteOne) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto doc1 = BSON("encrypted" << "a"); auto doc2 = BSON("encrypted" @@ -1536,7 +1534,6 @@ TEST_F(FleTagsTest, InsertTwoDifferentAndDeleteOne) { } TEST_F(FleTagsTest, InsertAndUpdate) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto doc1 = BSON("encrypted" << "a"); auto doc2 = BSON("encrypted" @@ -1582,7 +1579,6 @@ TEST_F(FleTagsTest, ContentionFactor) { } TEST_F(FleTagsTest, MemoryLimit) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto doc = BSON("encrypted" << "a"); diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index 7800bdec863..d442d4eb35f 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -3888,16 +3888,6 @@ constexpr auto kInternalFleEq = "$_internalFleEq"_sd; ExpressionInternalFLEEqual::ExpressionInternalFLEEqual(ExpressionContext* const expCtx, boost::intrusive_ptr<Expression> field, - ConstDataRange serverToken, - int64_t contentionFactor, - ConstDataRange edcToken) - : Expression(expCtx, {std::move(field)}), - _evaluator(serverToken, contentionFactor, {edcToken}) { - expCtx->sbeCompatibility = SbeCompatibility::notCompatible; -} - -ExpressionInternalFLEEqual::ExpressionInternalFLEEqual(ExpressionContext* const expCtx, - boost::intrusive_ptr<Expression> field, ServerZerosEncryptionToken zerosToken) : Expression(expCtx, {std::move(field)}), _evaluatorV2({std::move(zerosToken)}) { expCtx->sbeCompatibility = SbeCompatibility::notCompatible; @@ -3911,33 +3901,6 @@ intrusive_ptr<Expression> ExpressionInternalFLEEqual::parse(ExpressionContext* c IDLParserContext ctx(kInternalFleEq); - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - auto fleEq = InternalFleEqStruct::parse(ctx, expr.Obj()); - - auto fieldExpr = Expression::parseOperand(expCtx, fleEq.getField().getElement(), vps); - - auto serverTokenPair = fromEncryptedConstDataRange(fleEq.getServerEncryptionToken()); - - uassert(6762901, - "Invalid server token", - serverTokenPair.first == EncryptedBinDataType::kFLE2TransientRaw && - serverTokenPair.second.length() == sizeof(PrfBlock)); - - auto edcTokenPair = fromEncryptedConstDataRange(fleEq.getEdcDerivedToken()); - - uassert(6762902, - "Invalid edc token", - edcTokenPair.first == EncryptedBinDataType::kFLE2TransientRaw && - edcTokenPair.second.length() == sizeof(PrfBlock)); - - - auto cf = fleEq.getMaxCounter(); - - return new ExpressionInternalFLEEqual( - expCtx, std::move(fieldExpr), serverTokenPair.second, cf, edcTokenPair.second); - } - auto fleEq = InternalFleEqStructV2::parse(ctx, expr.Obj()); auto fieldExpr = Expression::parseOperand(expCtx, fleEq.getField().getElement(), vps); @@ -3959,16 +3922,6 @@ Value toValue(const std::array<std::uint8_t, 32>& buf) { } Value ExpressionInternalFLEEqual::serialize(SerializationOptions options) const { - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return Value(Document{ - {kInternalFleEq, - Document{{"field", _children[0]->serialize(options)}, - {"edc", toValue(_evaluator.edcTokens()[0])}, - {"counter", Value(static_cast<long long>(_evaluator.contentionFactor()))}, - {"server", toValue(_evaluator.serverToken())}}}}); - } - return Value( Document{{kInternalFleEq, Document{{"field", _children[0]->serialize(options)}, @@ -3981,19 +3934,6 @@ Value ExpressionInternalFLEEqual::evaluate(const Document& root, Variables* vari return Value(BSONNULL); } - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return Value(_evaluator.evaluate( - fieldValue, - EncryptedBinDataType::kFLE2EqualityIndexedValue, - [](auto token, auto serverValue) { - auto swIndexed = EDCServerCollection::decryptAndParse(token, serverValue); - uassertStatusOK(swIndexed); - const auto& indexed = swIndexed.getValue(); - return std::vector<EDCDerivedFromDataTokenAndContentionFactorToken>{indexed.edc}; - })); - } - return Value(_evaluatorV2.evaluate( fieldValue, EncryptedBinDataType::kFLE2EqualityIndexedValueV2, [](auto serverValue) { auto swParsedFields = @@ -4013,15 +3953,6 @@ const char* ExpressionInternalFLEEqual::getOpName() const { constexpr auto kInternalFleBetween = "$_internalFleBetween"_sd; -ExpressionInternalFLEBetween::ExpressionInternalFLEBetween(ExpressionContext* const expCtx, - boost::intrusive_ptr<Expression> field, - ConstDataRange serverToken, - int64_t contentionFactor, - std::vector<ConstDataRange> edcTokens) - : Expression(expCtx, {std::move(field)}), _evaluator(serverToken, contentionFactor, edcTokens) { - expCtx->sbeCompatibility = SbeCompatibility::notCompatible; -} - ExpressionInternalFLEBetween::ExpressionInternalFLEBetween( ExpressionContext* const expCtx, boost::intrusive_ptr<Expression> field, @@ -4037,36 +3968,6 @@ intrusive_ptr<Expression> ExpressionInternalFLEBetween::parse(ExpressionContext* const VariablesParseState& vps) { IDLParserContext ctx(kInternalFleBetween); - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - auto fleBetween = InternalFleBetweenStruct::parse(ctx, expr.Obj()); - - auto fieldExpr = Expression::parseOperand(expCtx, fleBetween.getField().getElement(), vps); - - auto serverTokenPair = fromEncryptedConstDataRange(fleBetween.getServerEncryptionToken()); - - uassert(6762904, - "Invalid server token", - serverTokenPair.first == EncryptedBinDataType::kFLE2TransientRaw && - serverTokenPair.second.length() == sizeof(PrfBlock)); - - std::vector<ConstDataRange> edcTokens; - for (auto& elem : fleBetween.getEdcDerivedTokens()) { - auto [first, second] = fromEncryptedConstDataRange(elem); - uassert(6762905, - "Invalid edc token", - first == EncryptedBinDataType::kFLE2TransientRaw && - second.length() == sizeof(PrfBlock)); - edcTokens.push_back(second); - } - - auto cf = fleBetween.getMaxCounter(); - uassert(6762906, "Contention factor must be between 0 and 10000", cf >= 0 && cf < 10000); - - return new ExpressionInternalFLEBetween( - expCtx, std::move(fieldExpr), serverTokenPair.second, cf, edcTokens); - } - auto fleBetween = InternalFleBetweenStructV2::parse(ctx, expr.Obj()); auto fieldExpr = Expression::parseOperand(expCtx, fleBetween.getField().getElement(), vps); @@ -4090,21 +3991,6 @@ intrusive_ptr<Expression> ExpressionInternalFLEBetween::parse(ExpressionContext* } Value ExpressionInternalFLEBetween::serialize(SerializationOptions options) const { - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - std::vector<Value> edcValues; - edcValues.reserve(_evaluator.edcTokens().size()); - for (auto& token : _evaluator.edcTokens()) { - edcValues.push_back(toValue(PrfBlockfromCDR(token))); - } - return Value(Document{ - {kInternalFleBetween, - Document{{"field", _children[0]->serialize(options)}, - {"edc", Value(edcValues)}, - {"counter", Value(static_cast<long long>(_evaluator.contentionFactor()))}, - {"server", toValue(_evaluator.serverToken())}}}}); - } - std::vector<Value> serverDerivedValues; serverDerivedValues.reserve(_evaluatorV2.zerosDecryptionTokens().size()); for (auto& token : _evaluatorV2.zerosDecryptionTokens()) { @@ -4121,22 +4007,6 @@ Value ExpressionInternalFLEBetween::evaluate(const Document& root, Variables* va return Value(BSONNULL); } - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return Value(_evaluator.evaluate( - fieldValue, - EncryptedBinDataType::kFLE2RangeIndexedValue, - [](auto token, auto serverValue) { - auto indexed = - uassertStatusOK(EDCServerCollection::decryptAndParseRange(token, serverValue)); - std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> edcTokens; - edcTokens.reserve(indexed.tokens.size()); - for (auto& edge : indexed.tokens) { - edcTokens.push_back(std::move(edge.edc)); - } - return edcTokens; - })); - } return Value(_evaluatorV2.evaluate( fieldValue, EncryptedBinDataType::kFLE2RangeIndexedValueV2, [](auto serverValue) { auto swParsedFields = diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h index b877494b41c..5b37d162fe8 100644 --- a/src/mongo/db/pipeline/expression.h +++ b/src/mongo/db/pipeline/expression.h @@ -2357,13 +2357,6 @@ public: class ExpressionInternalFLEEqual final : public Expression { public: - // TODO: SERVER-73303 delete constructor when v2 is enabled by default - ExpressionInternalFLEEqual(ExpressionContext* expCtx, - boost::intrusive_ptr<Expression> field, - ConstDataRange serverToken, - int64_t contentionFactor, - ConstDataRange edcToken); - ExpressionInternalFLEEqual(ExpressionContext* expCtx, boost::intrusive_ptr<Expression> field, ServerZerosEncryptionToken zerosToken); @@ -2386,19 +2379,11 @@ public: } private: - EncryptedPredicateEvaluator _evaluator; EncryptedPredicateEvaluatorV2 _evaluatorV2; }; class ExpressionInternalFLEBetween final : public Expression { public: - // TODO: SERVER-73303 delete constructor when v2 is enabled by default - ExpressionInternalFLEBetween(ExpressionContext* expCtx, - boost::intrusive_ptr<Expression> field, - ConstDataRange serverToken, - int64_t contentionFactor, - std::vector<ConstDataRange> edcTokens); - ExpressionInternalFLEBetween(ExpressionContext* expCtx, boost::intrusive_ptr<Expression> field, std::vector<ServerZerosEncryptionToken> serverTokens); @@ -2421,7 +2406,6 @@ public: } private: - EncryptedPredicateEvaluator _evaluator; EncryptedPredicateEvaluatorV2 _evaluatorV2; }; diff --git a/src/mongo/db/pipeline/expression_parser.idl b/src/mongo/db/pipeline/expression_parser.idl index a3360d39819..8ed67cf2f43 100644 --- a/src/mongo/db/pipeline/expression_parser.idl +++ b/src/mongo/db/pipeline/expression_parser.idl @@ -32,31 +32,6 @@ imports: - "mongo/db/basic_types.idl" structs: - # TODO: SERVER-73303 remove when v2 is enabled by default - InternalFleEqStruct: - description: "Struct for $_internalFleEq" - strict: true - fields: - field: - description: "Expression" - type: IDLAnyType - cpp_name: field - edc: - description: "EDCDerivedFromDataToken" - type: bindata_encrypt - cpp_name: edcDerivedToken - server: - description: "ServerDataEncryptionLevel1Token" - type: bindata_encrypt - cpp_name: serverEncryptionToken - counter: - description: "Queryable Encryption max counter" - type: long - cpp_name: maxCounter - validator: - gte: 0 - lt: 10000 - InternalFleEqStructV2: description: "Struct for $_internalFleEq" strict: true @@ -70,28 +45,6 @@ structs: type: bindata_encrypt cpp_name: serverZerosEncryptionToken - # TODO: SERVER-73303 remove when v2 is enabled by default - InternalFleBetweenStruct: - description: "Struct for $_internalFleBetween" - strict: true - fields: - field: - description: "Expression" - type: IDLAnyType - cpp_name: field - edc: - description: "EDCDerivedFromDataTokens" - type: array<bindata_encrypt> - cpp_name: edcDerivedTokens - server: - description: "ServerDataEncryptionLevel1Token" - type: bindata_encrypt - cpp_name: serverEncryptionToken - counter: - description: "Queryable Encryption max counter" - type: long - cpp_name: maxCounter - InternalFleBetweenStructV2: description: "Struct for $_internalFleBetween" strict: true diff --git a/src/mongo/db/pipeline/expression_test.cpp b/src/mongo/db/pipeline/expression_test.cpp index ba1cf96b27c..0b1631d6bff 100644 --- a/src/mongo/db/pipeline/expression_test.cpp +++ b/src/mongo/db/pipeline/expression_test.cpp @@ -4298,7 +4298,6 @@ TEST(ExpressionFLETest, BadInputs) { // Test we return true if it matches TEST(ExpressionFLETest, TestBinData) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto expCtx = ExpressionContextForTest(); auto vps = expCtx.variablesParseState; @@ -4347,7 +4346,6 @@ TEST(ExpressionFLETest, TestBinData) { } TEST(ExpressionFLETest, TestBinData_RoundTrip) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto expCtx = ExpressionContextForTest(); auto vps = expCtx.variablesParseState; @@ -4392,7 +4390,6 @@ TEST(ExpressionFLETest, TestBinData_RoundTrip) { } TEST(ExpressionFLETest, ParseAndSerializeBetween) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); auto expCtx = ExpressionContextForTest(); auto vps = expCtx.variablesParseState; diff --git a/src/mongo/db/query/fle/encrypted_predicate_test_fixtures.h b/src/mongo/db/query/fle/encrypted_predicate_test_fixtures.h index b1f39f80e61..d83d5056ba3 100644 --- a/src/mongo/db/query/fle/encrypted_predicate_test_fixtures.h +++ b/src/mongo/db/query/fle/encrypted_predicate_test_fixtures.h @@ -49,10 +49,6 @@ using TagMap = std::map<std::pair<StringData, int>, std::vector<PrfBlock>>; class MockServerRewrite : public QueryRewriterInterface { public: MockServerRewrite() : _expCtx((new ExpressionContextForTest())) { - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabledAndIgnoreFCVUnsafe()) { - _mockOptionalNss = NamespaceString({"mock"_sd}); - return; - } _mockOptionalNss = boost::none; } EncryptedCollScanMode getEncryptedCollScanMode() const override { @@ -72,9 +68,6 @@ public: const NamespaceString& getESCNss() const override { return _mockNss; } - const boost::optional<NamespaceString>& getECCNss() const override { - return _mockOptionalNss; - } private: diff --git a/src/mongo/db/query/fle/equality_predicate.cpp b/src/mongo/db/query/fle/equality_predicate.cpp index a7ff4b5ae92..3da59874b7a 100644 --- a/src/mongo/db/query/fle/equality_predicate.cpp +++ b/src/mongo/db/query/fle/equality_predicate.cpp @@ -48,17 +48,6 @@ REGISTER_ENCRYPTED_AGG_PREDICATE_REWRITE(ExpressionIn, EqualityPredicate); std::vector<PrfBlock> EqualityPredicate::generateTags(BSONValue payload) const { ParsedFindEqualityPayload tokens = parseFindPayload<ParsedFindEqualityPayload>(payload); - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - invariant(_rewriter->getECCNss()); - return readTags(_rewriter->getTagQueryInterface(), - _rewriter->getESCNss(), - _rewriter->getECCNss().get(), - tokens.escToken, - tokens.eccToken, - tokens.edcToken, - tokens.maxCounter); - } - return readTags(_rewriter->getTagQueryInterface(), _rewriter->getESCNss(), tokens.escToken, @@ -118,24 +107,6 @@ boost::intrusive_ptr<ExpressionInternalFLEEqual> generateFleEqualMatch(StringDat ExpressionContext* expCtx) { auto tokens = ParsedFindEqualityPayload(ffp); - // TODO: SERVER-73303 refactor when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - // Generate { $_internalFleEq: { field: "$field_name", server: f_3, counter: cm, edc: k_EDC] - // } - // } - uassert(6672401, - "Missing required field server encryption token in find payload", - tokens.serverToken.has_value()); - - return make_intrusive<ExpressionInternalFLEEqual>( - expCtx, - ExpressionFieldPath::createPathFromString( - expCtx, path.toString(), expCtx->variablesParseState), - tokens.serverToken.value().data, - tokens.maxCounter.value_or(0LL), - tokens.edcToken.data); - } - // Generate { $_internalFleEq: { field: "$field_name", server: F_s[ f, 2, v, 2 ] } return make_intrusive<ExpressionInternalFLEEqual>( expCtx, @@ -152,23 +123,6 @@ std::unique_ptr<ExpressionInternalFLEEqual> generateFleEqualMatchUnique(StringDa ExpressionContext* expCtx) { auto tokens = ParsedFindEqualityPayload(ffp); - // TODO: SERVER-73303 refactor when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - // Generate { $_internalFleEq: { field: "$field_name", server: f_3, counter: cm, edc: k_EDC] - // } } - uassert(6672419, - "Missing required field server encryption token in find payload", - tokens.serverToken.has_value()); - - return std::make_unique<ExpressionInternalFLEEqual>( - expCtx, - ExpressionFieldPath::createPathFromString( - expCtx, path.toString(), expCtx->variablesParseState), - tokens.serverToken.value().data, - tokens.maxCounter.value_or(0LL), - tokens.edcToken.data); - } - // Generate { $_internalFleEq: { field: "$field_name", server: F_s[ f, 2, v, 2 ] } return std::make_unique<ExpressionInternalFLEEqual>( expCtx, diff --git a/src/mongo/db/query/fle/equality_predicate.h b/src/mongo/db/query/fle/equality_predicate.h index 0f3e3c05d0e..e57fcd7f1a9 100644 --- a/src/mongo/db/query/fle/equality_predicate.h +++ b/src/mongo/db/query/fle/equality_predicate.h @@ -52,20 +52,13 @@ protected: std::unique_ptr<Expression> rewriteToRuntimeComparison(Expression* expr) const override; bool isDeprecatedPayloadType(EncryptedBinDataType type) const override { - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return false; - } return type == EncryptedBinDataType::kFLE2FindEqualityPayload || type == EncryptedBinDataType::kFLE2UnindexedEncryptedValue; } private: EncryptedBinDataType encryptedBinDataType() const override { - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return EncryptedBinDataType::kFLE2FindEqualityPayload; - } + return EncryptedBinDataType::kFLE2FindEqualityPayloadV2; } diff --git a/src/mongo/db/query/fle/query_rewriter.h b/src/mongo/db/query/fle/query_rewriter.h index fdb51cf14d7..c200df79ced 100644 --- a/src/mongo/db/query/fle/query_rewriter.h +++ b/src/mongo/db/query/fle/query_rewriter.h @@ -51,38 +51,7 @@ namespace mongo::fle { class QueryRewriter : public QueryRewriterInterface { public: /** - * Takes in references to collection readers for the ESC and ECC that are used during tag - * computation. - * - * TODO SERVER-73303 Remove when V2 is enabled. - */ - QueryRewriter(boost::intrusive_ptr<ExpressionContext> expCtx, - FLETagQueryInterface* tagQueryInterface, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc, - EncryptedCollScanModeAllowed mode = EncryptedCollScanModeAllowed::kAllow) - : _expCtx(expCtx), - _exprRewrites(aggPredicateRewriteMap), - _matchRewrites(matchPredicateRewriteMap), - _nssEsc(nssEsc), - _nssEcc(nssEcc), - _tagQueryInterface(tagQueryInterface) { - - if (internalQueryFLEAlwaysUseEncryptedCollScanMode.load()) { - _mode = EncryptedCollScanMode::kForceAlways; - } - - if (mode == EncryptedCollScanModeAllowed::kDisallow) { - _mode = EncryptedCollScanMode::kDisallow; - } - - // This isn't the "real" query so we don't want to increment Expression - // counters here. - _expCtx->stopExpressionCounters(); - } - - /** - * Takes in references to collection readers for the ESC and ECC that are used during tag + * Takes in references to collection readers for the ESC that are used during tag * computation. */ QueryRewriter(boost::intrusive_ptr<ExpressionContext> expCtx, @@ -93,7 +62,6 @@ public: _exprRewrites(aggPredicateRewriteMap), _matchRewrites(matchPredicateRewriteMap), _nssEsc(nssEsc), - _nssEcc(boost::none), _tagQueryInterface(tagQueryInterface) { if (internalQueryFLEAlwaysUseEncryptedCollScanMode.load()) { @@ -152,22 +120,17 @@ public: const NamespaceString& getESCNss() const override { return _nssEsc; } - const boost::optional<NamespaceString>& getECCNss() const override { - return _nssEcc; - } protected: // This constructor should only be used for mocks in testing. QueryRewriter(boost::intrusive_ptr<ExpressionContext> expCtx, const NamespaceString& nssEsc, - const NamespaceString& nssEcc, const ExpressionToRewriteMap& exprRewrites, const MatchTypeToRewriteMap& matchRewrites) : _expCtx(expCtx), _exprRewrites(exprRewrites), _matchRewrites(matchRewrites), _nssEsc(nssEsc), - _nssEcc(nssEcc), _tagQueryInterface(nullptr) {} private: @@ -187,7 +150,6 @@ private: const ExpressionToRewriteMap& _exprRewrites; const MatchTypeToRewriteMap& _matchRewrites; const NamespaceString& _nssEsc; - const boost::optional<NamespaceString> _nssEcc; FLETagQueryInterface* _tagQueryInterface; }; } // namespace mongo::fle diff --git a/src/mongo/db/query/fle/query_rewriter_interface.h b/src/mongo/db/query/fle/query_rewriter_interface.h index 7576609a46e..735f1954d3a 100644 --- a/src/mongo/db/query/fle/query_rewriter_interface.h +++ b/src/mongo/db/query/fle/query_rewriter_interface.h @@ -67,7 +67,6 @@ public: virtual ~QueryRewriterInterface() {} virtual FLETagQueryInterface* getTagQueryInterface() const = 0; virtual const NamespaceString& getESCNss() const = 0; - virtual const boost::optional<NamespaceString>& getECCNss() const = 0; virtual EncryptedCollScanMode getEncryptedCollScanMode() const = 0; virtual ExpressionContext* getExpressionContext() const = 0; diff --git a/src/mongo/db/query/fle/query_rewriter_test.cpp b/src/mongo/db/query/fle/query_rewriter_test.cpp index 9c2bc41a28c..7c985cc22f6 100644 --- a/src/mongo/db/query/fle/query_rewriter_test.cpp +++ b/src/mongo/db/query/fle/query_rewriter_test.cpp @@ -241,7 +241,7 @@ public: fle::MatchTypeToRewriteMap* matchRewrites, const NamespaceString& mockNss) : fle::QueryRewriter( - new ExpressionContextForTest(), mockNss, mockNss, *exprRewrites, *matchRewrites) { + new ExpressionContextForTest(), mockNss, *exprRewrites, *matchRewrites) { setMockRewriteMaps(*matchRewrites, *exprRewrites, _tags, _encryptedFields); } diff --git a/src/mongo/db/query/fle/range_predicate.cpp b/src/mongo/db/query/fle/range_predicate.cpp index 11792c8415d..ab271cc24b6 100644 --- a/src/mongo/db/query/fle/range_predicate.cpp +++ b/src/mongo/db/query/fle/range_predicate.cpp @@ -100,20 +100,6 @@ std::unique_ptr<ExpressionInternalFLEBetween> RangePredicate::fleBetweenFromPayl tassert(7030501, "$internalFleBetween can only be generated from a non-stub payload.", !payload.isStub()); - // TODO: SERVER-73303 refactor when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - auto cm = payload.maxCounter; - ServerDataEncryptionLevel1Token serverToken = std::move(payload.serverToken); - std::vector<ConstDataRange> edcTokens; - std::transform(std::make_move_iterator(payload.edges.value().begin()), - std::make_move_iterator(payload.edges.value().end()), - std::back_inserter(edcTokens), - [](FLEFindEdgeTokenSet&& edge) { return edge.edc.toCDR(); }); - - auto* expCtx = _rewriter->getExpressionContext(); - return std::make_unique<ExpressionInternalFLEBetween>( - expCtx, fieldpath, serverToken.toCDR(), cm, std::move(edcTokens)); - } std::vector<ServerZerosEncryptionToken> serverZerosTokens; serverZerosTokens.reserve(payload.edges.value().size()); @@ -137,25 +123,6 @@ std::vector<PrfBlock> RangePredicate::generateTags(BSONValue payload) const { std::vector<PrfBlock> tags; tassert(7030500, "Must generate tags from a non-stub payload.", !parsedPayload.isStub()); - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - invariant(_rewriter->getECCNss()); - - for (auto& edge : parsedPayload.edges.value()) { - auto tagsForEdge = readTags(_rewriter->getTagQueryInterface(), - _rewriter->getESCNss(), - _rewriter->getECCNss().get(), - edge.esc, - edge.ecc, - edge.edc, - parsedPayload.maxCounter); - tags.insert(tags.end(), - std::make_move_iterator(tagsForEdge.begin()), - std::make_move_iterator(tagsForEdge.end())); - } - return tags; - } - // TODO - do batch generation of tags here for (auto& edge : parsedPayload.edges.value()) { auto tagsForEdge = readTags(_rewriter->getTagQueryInterface(), diff --git a/src/mongo/db/query/fle/range_predicate.h b/src/mongo/db/query/fle/range_predicate.h index db13c4492b1..865905c9863 100644 --- a/src/mongo/db/query/fle/range_predicate.h +++ b/src/mongo/db/query/fle/range_predicate.h @@ -61,20 +61,12 @@ protected: } bool isDeprecatedPayloadType(EncryptedBinDataType type) const override { - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return false; - } return type == EncryptedBinDataType::kFLE2FindRangePayload || type == EncryptedBinDataType::kFLE2UnindexedEncryptedValue; } private: EncryptedBinDataType encryptedBinDataType() const override { - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return EncryptedBinDataType::kFLE2FindRangePayload; - } return EncryptedBinDataType::kFLE2FindRangePayloadV2; } /** diff --git a/src/mongo/db/query/fle/range_validator.cpp b/src/mongo/db/query/fle/range_validator.cpp index e355199a4e8..0151c2e3052 100644 --- a/src/mongo/db/query/fle/range_validator.cpp +++ b/src/mongo/db/query/fle/range_validator.cpp @@ -43,9 +43,6 @@ namespace { template <class T> bool hasRangeTypeToValidate(const T& value) { auto typeToValidate = EncryptedBinDataType::kFLE2FindRangePayloadV2; - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - typeToValidate = EncryptedBinDataType::kFLE2FindRangePayload; - } return isPayloadOfType(typeToValidate, value); } diff --git a/src/mongo/db/query/fle/server_rewrite.cpp b/src/mongo/db/query/fle/server_rewrite.cpp index 1360b761c18..7626e9f4518 100644 --- a/src/mongo/db/query/fle/server_rewrite.cpp +++ b/src/mongo/db/query/fle/server_rewrite.cpp @@ -121,20 +121,6 @@ REGISTER_DOCUMENT_SOURCE_FLE_REWRITER(DocumentSourceMatch, rewriteMatch); REGISTER_DOCUMENT_SOURCE_FLE_REWRITER(DocumentSourceGeoNear, rewriteGeoNear); REGISTER_DOCUMENT_SOURCE_FLE_REWRITER(DocumentSourceGraphLookUp, rewriteGraphLookUp); -BSONObj rewriteEncryptedFilter(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc, - boost::intrusive_ptr<ExpressionContext> expCtx, - BSONObj filter, - EncryptedCollScanModeAllowed mode) { - - if (auto rewritten = - QueryRewriter(expCtx, queryImpl, nssEsc, nssEcc, mode).rewriteMatchExpression(filter)) { - return rewritten.value(); - } - - return filter; -} BSONObj rewriteEncryptedFilterV2(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc, @@ -159,23 +145,14 @@ public: : expCtx(expCtx), dbName(nss.dbName()) { auto efc = EncryptionInformationHelpers::getAndValidateSchema(nss, encryptInfo); esc = efc.getEscCollection()->toString(); - if (efc.getEccCollection()) { - ecc = efc.getEccCollection()->toString(); - } else { - ecc = boost::none; - } } virtual ~RewriteBase(){}; - virtual void doRewrite(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc){}; - virtual void doRewrite(FLEQueryInterface* queryImpl, const NamespaceString& nssEsc){}; + virtual void doRewrite(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc){}; boost::intrusive_ptr<ExpressionContext> expCtx; std::string esc; - boost::optional<std::string> ecc; DatabaseName dbName; }; @@ -188,18 +165,8 @@ public: : RewriteBase(toRewrite->getContext(), nss, encryptInfo), pipeline(std::move(toRewrite)) {} ~PipelineRewrite(){}; - void doRewrite(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc) final { - auto rewriter = QueryRewriter(expCtx, queryImpl, nssEsc, nssEcc); - for (auto&& source : pipeline->getSources()) { - if (stageRewriterMap.find(typeid(*source)) != stageRewriterMap.end()) { - stageRewriterMap[typeid(*source)](&rewriter, source.get()); - } - } - } - void doRewrite(FLEQueryInterface* queryImpl, const NamespaceString& nssEsc) final { + void doRewrite(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc) final { auto rewriter = QueryRewriter(expCtx, queryImpl, nssEsc); for (auto&& source : pipeline->getSources()) { if (stageRewriterMap.find(typeid(*source)) != stageRewriterMap.end()) { @@ -227,14 +194,8 @@ public: : RewriteBase(expCtx, nss, encryptInfo), userFilter(toRewrite), _mode(mode) {} ~FilterRewrite(){}; - void doRewrite(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc) final { - rewrittenFilter = - rewriteEncryptedFilter(queryImpl, nssEsc, nssEcc, expCtx, userFilter, _mode); - } - void doRewrite(FLEQueryInterface* queryImpl, const NamespaceString& nssEsc) final { + void doRewrite(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc) final { rewrittenFilter = rewriteEncryptedFilterV2(queryImpl, nssEsc, expCtx, userFilter, _mode); } @@ -251,19 +212,16 @@ void doFLERewriteInTxn(OperationContext* opCtx, std::shared_ptr<RewriteBase> sharedBlock, GetTxnCallback getTxn) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - // This code path only works if we are NOT running in a a transaction. - // if breaks us off of the current optctx readconcern and other settings - // - if (!opCtx->inMultiDocumentTransaction()) { - NamespaceString nssEsc(NamespaceStringUtil::parseNamespaceFromRequest( - sharedBlock->dbName, sharedBlock->esc)); - NamespaceString nssEcc; // Ignored in V2 - FLETagNoTXNQuery queryInterface(opCtx); + // This code path only works if we are NOT running in a a transaction. + // if breaks us off of the current optctx readconcern and other settings + // + if (!opCtx->inMultiDocumentTransaction()) { + NamespaceString nssEsc( + NamespaceStringUtil::parseNamespaceFromRequest(sharedBlock->dbName, sharedBlock->esc)); + FLETagNoTXNQuery queryInterface(opCtx); - sharedBlock->doRewrite(&queryInterface, nssEsc, nssEcc); - return; - } + sharedBlock->doRewrite(&queryInterface, nssEsc); + return; } auto txn = getTxn(opCtx); @@ -276,14 +234,7 @@ void doFLERewriteInTxn(OperationContext* opCtx, auto queryInterface = FLEQueryInterfaceImpl(txnClient, getGlobalServiceContext()); // Rewrite the MatchExpression. - if (sharedBlock->ecc) { - sharedBlock->doRewrite(&queryInterface, - nssEsc, - NamespaceStringUtil::parseNamespaceFromRequest( - sharedBlock->dbName, sharedBlock->ecc.get())); - } else { - sharedBlock->doRewrite(&queryInterface, nssEsc); - } + sharedBlock->doRewrite(&queryInterface, nssEsc); return SemiFuture<void>::makeReady(); }); @@ -303,12 +254,6 @@ BSONObj rewriteEncryptedFilterInsideTxn(FLETagQueryInterface* queryImpl, NamespaceString nssEsc( NamespaceStringUtil::parseNamespaceFromRequest(dbName, efc.getEscCollection().value())); - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - NamespaceString nssEcc( - NamespaceStringUtil::parseNamespaceFromRequest(dbName, efc.getEccCollection().value())); - return rewriteEncryptedFilter(queryImpl, nssEsc, nssEcc, expCtx, filter, mode); - } - return rewriteEncryptedFilterV2(queryImpl, nssEsc, expCtx, filter, mode); } diff --git a/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp b/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp index a24536e1400..3b4c63f714f 100644 --- a/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp +++ b/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp @@ -223,23 +223,10 @@ bool doRenameOperation(const CompactionState& state, CompactStats doCompactOperationPre70Compatible( const CompactStructuredEncryptionDataStatePre70Compatible& state) { - if (state.getSkipCompact()) { - LOGV2_DEBUG(6517005, 1, "Skipping compaction"); - CompactStats stats({}, {}); - stats.setEcc({}); - return stats; - } - - EncryptedStateCollectionsNamespaces namespaces; - namespaces.edcNss = state.getId().getNss(); - namespaces.eccNss = state.getEccNss(); - namespaces.escNss = state.getEscNss(); - namespaces.ecocNss = state.getEcocNss(); - namespaces.ecocRenameNss = state.getEcocRenameNss(); - auto opCtx = cc().makeOperationContext(); - CompactStructuredEncryptionData request(namespaces.edcNss, state.getCompactionTokens()); - - return processFLECompact(opCtx.get(), request, &getTransactionWithRetriesForMongoS, namespaces); + LOGV2_DEBUG(6517005, 1, "Skipping compaction"); + CompactStats stats({}, {}); + stats.setEcc({}); + return stats; } void doCompactOperation(const CompactStructuredEncryptionDataState& state, diff --git a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp index b5473d9fd2f..bca40634ef6 100644 --- a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp +++ b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp @@ -82,7 +82,6 @@ public: using InvocationBase::InvocationBase; Reply typedRun(OperationContext* opCtx) { - bool useV1Protocol = false; CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; @@ -90,38 +89,19 @@ public: [&]() -> std::shared_ptr<ShardingDDLCoordinatorService::Instance> { FixedFCVRegion fixedFcvRegion(opCtx); - useV1Protocol = !gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility); - if (useV1Protocol) { - auto compact = makeRequestV1(opCtx); - if (!compact) { - return nullptr; - } - return ShardingDDLCoordinatorService::getService(opCtx)->getOrCreateInstance( - opCtx, compact->toBSON()); - } + // TODO: SERVER-68373 Remove once 7.0 becomes last LTS + uassert(7330300, + "The preview version of compactStructuredEncryptionData is no longer " + "supported in this binary version", + gFeatureFlagFLE2CompactForProtocolV2.isEnabled( + serverGlobalParams.featureCompatibility)); + auto compact = makeRequest(opCtx); return ShardingDDLCoordinatorService::getService(opCtx)->getOrCreateInstance( opCtx, compact.toBSON()); }(); - if (!compactCoordinator) { - // Nothing to do. - LOGV2(6548305, "Skipping compaction as there is no ECOC collection to compact"); - CompactStats stats({}, {}); - if (useV1Protocol) { - stats.setEcc(ECStats{}); - } - return stats; - } - - if (useV1Protocol) { - return checked_pointer_cast< - CompactStructuredEncryptionDataCoordinatorPre70Compatible>( - compactCoordinator) - ->getResponse(opCtx); - } return checked_pointer_cast<CompactStructuredEncryptionDataCoordinator>( compactCoordinator) ->getResponse(opCtx); @@ -165,58 +145,6 @@ public: return compact; } - // TODO: SERVER-68373 Remove once 7.0 becomes last LTS - boost::optional<CompactStructuredEncryptionDataStatePre70Compatible> makeRequestV1( - OperationContext* opCtx) { - const auto& req = request(); - const auto& nss = req.getNamespace(); - - AutoGetCollection baseColl(opCtx, nss, MODE_IX); - uassert(ErrorCodes::NamespaceNotFound, - str::stream() << "Unknown collection: " << nss, - baseColl.getCollection()); - - validateCompactRequest(req, *(baseColl.getCollection().get())); - - auto namespaces = - uassertStatusOK(EncryptedStateCollectionsNamespaces::createFromDataCollection( - *(baseColl.getCollection().get()))); - - AutoGetCollection ecocColl(opCtx, namespaces.ecocNss, MODE_IX); - AutoGetCollection ecocTempColl(opCtx, namespaces.ecocRenameNss, MODE_IX); - - if (!ecocColl.getCollection() && !ecocTempColl.getCollection()) { - return boost::none; - } - - CompactStructuredEncryptionDataStatePre70Compatible compact; - auto coordinatorType = - DDLCoordinatorTypeEnum::kCompactStructuredEncryptionDataPre70Compatible; - - if (!gFeatureFlagUseNewCompactStructuredEncryptionDataCoordinator.isEnabled( - serverGlobalParams.featureCompatibility)) { - // TODO SERVER-68373 remove once 7.0 becomes last LTS - coordinatorType = - DDLCoordinatorTypeEnum::kCompactStructuredEncryptionDataPre61Compatible; - } - - if (ecocColl.getCollection()) { - compact.setEcocUuid(ecocColl->uuid()); - } - if (ecocTempColl.getCollection()) { - compact.setEcocRenameUuid(ecocTempColl->uuid()); - } - - compact.setShardingDDLCoordinatorMetadata({{nss, coordinatorType}}); - compact.setEscNss(namespaces.escNss); - compact.setEccNss(namespaces.eccNss); - compact.setEcocNss(namespaces.ecocNss); - compact.setEcocRenameNss(namespaces.ecocRenameNss); - compact.setCompactionTokens(req.getCompactionTokens().getOwned()); - - return compact; - } - NamespaceString ns() const override { return request().getNamespace(); } diff --git a/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp b/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp index c0d2acb22c1..91b6aa8cbfc 100644 --- a/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp +++ b/src/mongo/s/commands/cluster_fle2_get_count_info_cmd.cpp @@ -102,10 +102,6 @@ ClusterGetQueryableEncryptionCountInfoCmd::Invocation::typedRun(OperationContext CurOp::get(opCtx)->debug().shouldOmitDiagnosticInformation = true; - uassert(741502, - "FeatureFlagFLE2ProtocolVersion2 is not enabled", - gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)); - auto nss = request().getNamespace(); const auto dbInfo = uassertStatusOK(Grid::get(opCtx)->catalogCache()->getDatabase(opCtx, nss.db())); |