diff options
author | Shreyas Kalyan <shreyas.kalyan@mongodb.com> | 2023-04-04 17:48:52 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-04-05 16:02:51 +0000 |
commit | 858c7ae1576bdb897a6c8e8162a180aac36dcda8 (patch) | |
tree | 878842e5ef0d8d22f073062ca9e1ac213a69c049 /src/mongo/crypto | |
parent | 74641e01217dfcae9a7d14fd4dcc329af2d489c7 (diff) | |
download | mongo-858c7ae1576bdb897a6c8e8162a180aac36dcda8.tar.gz |
SERVER-73303 Remove Queryable Encryption v1 Code
Diffstat (limited to 'src/mongo/crypto')
-rw-r--r-- | src/mongo/crypto/encryption_fields.idl | 4 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto.cpp | 1608 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto.h | 301 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto_predicate.h | 39 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto_test.cpp | 303 | ||||
-rw-r--r-- | src/mongo/crypto/fle_field_schema.idl | 158 | ||||
-rw-r--r-- | src/mongo/crypto/fle_stats.h | 7 | ||||
-rw-r--r-- | src/mongo/crypto/fle_stats.idl | 4 | ||||
-rw-r--r-- | src/mongo/crypto/fle_tags.cpp | 150 | ||||
-rw-r--r-- | src/mongo/crypto/fle_tags.h | 25 |
10 files changed, 144 insertions, 2455 deletions
diff --git a/src/mongo/crypto/encryption_fields.idl b/src/mongo/crypto/encryption_fields.idl index cc89e8e6427..61c647ccacf 100644 --- a/src/mongo/crypto/encryption_fields.idl +++ b/src/mongo/crypto/encryption_fields.idl @@ -61,10 +61,10 @@ feature_flags: version: 6.2 cpp_varname: gFeatureFlagFLE2Range default: true - featureFlagFLE2ProtocolVersion2: + featureFlagFLE2CompactForProtocolV2: description: "Enable support for version 2 of Queryable Encryption wire protocol" version: 7.0 - cpp_varname: gFeatureFlagFLE2ProtocolVersion2 + cpp_varname: gFeatureFlagFLE2CompactForProtocolV2 default: true featureFlagFLE2CleanupCommand: description: "Enable support for the Cleanup Structured Encryption Data command" diff --git a/src/mongo/crypto/fle_crypto.cpp b/src/mongo/crypto/fle_crypto.cpp index f88f34a3779..c98541c5877 100644 --- a/src/mongo/crypto/fle_crypto.cpp +++ b/src/mongo/crypto/fle_crypto.cpp @@ -826,21 +826,6 @@ StatusWith<std::vector<uint8_t>> KeyIdAndValue::decrypt(FLEUserKey userKey, */ class EDCClientPayload { public: - // TODO: SERVER-73303 delete v1 functions when v2 is enabled by default - static FLE2InsertUpdatePayload parseInsertUpdatePayload(ConstDataRange cdr); - - static FLE2InsertUpdatePayload serializeInsertUpdatePayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t contentionFactor); - - - static FLE2InsertUpdatePayload serializeInsertUpdatePayloadForRange(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - FLE2RangeInsertSpec spec, - uint8_t sparsity, - uint64_t contentionFactor); - static FLE2InsertUpdatePayloadV2 parseInsertUpdatePayloadV2(ConstDataRange cdr); static FLE2InsertUpdatePayloadV2 serializeInsertUpdatePayloadV2(FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, @@ -854,70 +839,10 @@ public: uint64_t contentionFactor); }; - -FLE2InsertUpdatePayload EDCClientPayload::parseInsertUpdatePayload(ConstDataRange cdr) { - return parseFromCDR<FLE2InsertUpdatePayload>(cdr); -} - FLE2InsertUpdatePayloadV2 EDCClientPayload::parseInsertUpdatePayloadV2(ConstDataRange cdr) { return parseFromCDR<FLE2InsertUpdatePayloadV2>(cdr); } -FLE2InsertUpdatePayload EDCClientPayload::serializeInsertUpdatePayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t contentionFactor) { - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - EDCDerivedFromDataTokenAndContentionFactorToken edcDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateEDCDerivedFromDataTokenAndContentionFactorToken(edcDatakey, contentionFactor); - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, contentionFactor); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, contentionFactor); - - - FLE2InsertUpdatePayload iupayload; - - iupayload.setEdcDerivedToken(edcDataCounterkey.toCDR()); - iupayload.setEscDerivedToken(escDataCounterkey.toCDR()); - iupayload.setEccDerivedToken(eccDataCounterkey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - - - auto swCipherText = KeyIdAndValue::serialize(userKey, value, crypto::aesMode::ctr); - uassertStatusOK(swCipherText); - iupayload.setValue(swCipherText.getValue()); - iupayload.setType(element.type()); - - iupayload.setIndexKeyId(indexKey.keyId); - - return iupayload; -} - FLE2InsertUpdatePayloadV2 EDCClientPayload::serializeInsertUpdatePayloadV2( FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, @@ -1046,60 +971,6 @@ std::unique_ptr<Edges> getEdges(FLE2RangeInsertSpec spec, int sparsity) { } } -// TODO: SERVER-73303 delete when v2 is enabled by default -std::vector<EdgeTokenSet> getEdgeTokenSet(FLE2RangeInsertSpec spec, - int sparsity, - uint64_t contentionFactor, - const EDCToken& edcToken, - const ESCToken& escToken, - const ECCToken& eccToken, - const ECOCToken& ecocToken) { - const auto edges = getEdges(std::move(spec), sparsity); - const auto edgesList = edges->get(); - - std::vector<EdgeTokenSet> tokens; - - for (const auto& edge : edgesList) { - ConstDataRange cdr(edge.rawData(), edge.size()); - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, cdr); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, cdr); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, cdr); - - EDCDerivedFromDataTokenAndContentionFactorToken edcDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateEDCDerivedFromDataTokenAndContentionFactorToken(edcDatakey, - contentionFactor); - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, - contentionFactor); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, - contentionFactor); - - EdgeTokenSet ets; - - ets.setEdcDerivedToken(edcDataCounterkey.toCDR()); - ets.setEscDerivedToken(escDataCounterkey.toCDR()); - ets.setEccDerivedToken(eccDataCounterkey.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey) - .serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - ets.setEncryptedTokens(swEncryptedTokens.getValue()); - - tokens.push_back(ets); - } - - return tokens; -} - std::vector<EdgeTokenSetV2> getEdgeTokenSet( FLE2RangeInsertSpec spec, int sparsity, @@ -1150,69 +1021,6 @@ std::vector<EdgeTokenSetV2> getEdgeTokenSet( return tokens; } -FLE2InsertUpdatePayload EDCClientPayload::serializeInsertUpdatePayloadForRange( - FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - FLE2RangeInsertSpec spec, - uint8_t sparsity, - uint64_t contentionFactor) { - auto element = spec.getValue().getElement(); - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - EDCDerivedFromDataTokenAndContentionFactorToken edcDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateEDCDerivedFromDataTokenAndContentionFactorToken(edcDatakey, contentionFactor); - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, contentionFactor); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, contentionFactor); - - FLE2InsertUpdatePayload iupayload; - - iupayload.setEdcDerivedToken(edcDataCounterkey.toCDR()); - iupayload.setEscDerivedToken(escDataCounterkey.toCDR()); - iupayload.setEccDerivedToken(eccDataCounterkey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - - auto swCipherText = KeyIdAndValue::serialize(userKey, value, crypto::aesMode::ctr); - uassertStatusOK(swCipherText); - iupayload.setValue(swCipherText.getValue()); - iupayload.setType(element.type()); - - iupayload.setIndexKeyId(indexKey.keyId); - - auto edgeTokenSet = - getEdgeTokenSet(spec, sparsity, contentionFactor, edcToken, escToken, eccToken, ecocToken); - - if (!edgeTokenSet.empty()) { - iupayload.setEdgeTokenSet(edgeTokenSet); - } - - return iupayload; -} FLE2InsertUpdatePayloadV2 EDCClientPayload::serializeInsertUpdatePayloadV2ForRange( FLEIndexKeyAndId indexKey, @@ -1482,46 +1290,22 @@ void convertToFLE2Payload(FLEKeyVault* keyVault, if (ep.getType() == Fle2PlaceholderType::kInsert) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2( - indexKey, userKey, el, contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, - iupayload, - builder); - return; - } - - // TODO: SERVER-73303 delete when v2 is enabled by default - auto iupayload = EDCClientPayload::serializeInsertUpdatePayload( + auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2( indexKey, userKey, el, contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayload, + EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, iupayload, builder); - } else if (ep.getType() == Fle2PlaceholderType::kFind) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto findPayload = FLEClientCrypto::serializeFindPayloadV2( - indexKey, userKey, el, ep.getMaxContentionCounter()); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindEqualityPayloadV2, - findPayload, - builder); - return; - } + } else if (ep.getType() == Fle2PlaceholderType::kFind) { - // TODO: SERVER-73303 delete when v2 is enabled by default - auto findpayload = FLEClientCrypto::serializeFindPayload( + auto findPayload = FLEClientCrypto::serializeFindPayloadV2( indexKey, userKey, el, ep.getMaxContentionCounter()); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindEqualityPayload, - findpayload, + EncryptedBinDataType::kFLE2FindEqualityPayloadV2, + findPayload, builder); + } else { uasserted(6410100, "Unsupported Queryable Encryption placeholder type"); } @@ -1539,83 +1323,42 @@ void convertToFLE2Payload(FLEKeyVault* keyVault, << "' is not a valid type for Queryable Encryption Range", isFLE2RangeIndexedSupportedType(elRange.type())); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2ForRange( - indexKey, - userKey, - rangeInsertSpec, - ep.getSparsity().value(), // Enforced as non-optional in this case in IDL - contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, - iupayload, - builder); - return; - } - - // TODO: SERVER-73303 delete when v2 is enabled by default - auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadForRange( + auto iupayload = EDCClientPayload::serializeInsertUpdatePayloadV2ForRange( indexKey, userKey, rangeInsertSpec, ep.getSparsity().value(), // Enforced as non-optional in this case in IDL contentionFactor(ep)); - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2InsertUpdatePayload, + EncryptedBinDataType::kFLE2InsertUpdatePayloadV2, iupayload, builder); + } else if (ep.getType() == Fle2PlaceholderType::kFind) { IDLParserContext ctx("root"); auto rangeFindSpec = FLE2RangeFindSpec::parse(ctx, ep.getValue().getElement().Obj()); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - - auto findPayload = [&]() { - if (rangeFindSpec.getEdgesInfo().has_value()) { - auto edges = getMinCover(rangeFindSpec, ep.getSparsity().value()); - - return FLEClientCrypto::serializeFindRangePayloadV2( - indexKey, - userKey, - std::move(edges), - ep.getMaxContentionCounter(), - rangeFindSpec); - } else { - return FLEClientCrypto::serializeFindRangeStubV2(rangeFindSpec); - } - }(); - - toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindRangePayloadV2, - findPayload, - builder); - return; - } - - // TODO: SERVER-73303 delete when v2 is enabled by default - auto findpayload = [&]() { + auto findPayload = [&]() { if (rangeFindSpec.getEdgesInfo().has_value()) { auto edges = getMinCover(rangeFindSpec, ep.getSparsity().value()); - return FLEClientCrypto::serializeFindRangePayload( + return FLEClientCrypto::serializeFindRangePayloadV2( indexKey, userKey, std::move(edges), ep.getMaxContentionCounter(), rangeFindSpec); } else { - return FLEClientCrypto::serializeFindRangeStub(rangeFindSpec); + return FLEClientCrypto::serializeFindRangeStubV2(rangeFindSpec); } }(); toEncryptedBinData(fieldNameToSerialize, - EncryptedBinDataType::kFLE2FindRangePayload, - findpayload, + EncryptedBinDataType::kFLE2FindRangePayloadV2, + findPayload, builder); + } else { uasserted(6775303, "Unsupported Queryable Encryption placeholder type"); } @@ -1625,17 +1368,10 @@ void convertToFLE2Payload(FLEKeyVault* keyVault, << "' is not a valid type for Queryable Encryption", isFLE2UnindexedSupportedType(el.type())); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto payload = FLE2UnindexedEncryptedValueV2::serialize(userKey, el); - builder->appendBinData( - fieldNameToSerialize, payload.size(), BinDataType::Encrypt, payload.data()); - return; - } - - auto payload = FLE2UnindexedEncryptedValue::serialize(userKey, el); + auto payload = FLE2UnindexedEncryptedValueV2::serialize(userKey, el); builder->appendBinData( fieldNameToSerialize, payload.size(), BinDataType::Encrypt, payload.data()); + } else { uasserted(6338603, "Only Queryable Encryption style encryption placeholders are supported"); @@ -1656,17 +1392,12 @@ void parseAndVerifyInsertUpdatePayload(std::vector<EDCServerPayloadInfo>* pField EDCServerPayloadInfo payloadInfo; payloadInfo.fieldPathName = fieldPath.toString(); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7291901, - "Encountered a Queryable Encryption insert/update payload type that is no " - "longer supported", - type == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2); - auto iupayload = EDCClientPayload::parseInsertUpdatePayloadV2(subCdr); - payloadInfo.payload = VersionedInsertUpdatePayload(std::move(iupayload)); - } else { - auto iupayload = EDCClientPayload::parseInsertUpdatePayload(subCdr); - payloadInfo.payload = VersionedInsertUpdatePayload(std::move(iupayload)); - } + uassert(7291901, + "Encountered a Queryable Encryption insert/update payload type that is no " + "longer supported", + type == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2); + auto iupayload = EDCClientPayload::parseInsertUpdatePayloadV2(subCdr); + payloadInfo.payload = std::move(iupayload); auto bsonType = static_cast<BSONType>(payloadInfo.payload.getType()); @@ -1710,12 +1441,10 @@ void collectEDCServerInfo(std::vector<EDCServerPayloadInfo>* pFields, return; } else if (encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValue || encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7413901, - "Encountered a Queryable Encryption unindexed encrypted payload type that is " - "no longer supported", - encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2); - } + uassert(7413901, + "Encountered a Queryable Encryption unindexed encrypted payload type that is " + "no longer supported", + encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2); return; } uasserted(6373503, @@ -1741,148 +1470,72 @@ void convertServerPayload(ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); - if (encryptedTypeBinding == EncryptedBinDataType::kFLE2FindEqualityPayload || - encryptedTypeBinding == EncryptedBinDataType::kFLE2FindRangePayload || - encryptedTypeBinding == EncryptedBinDataType::kFLE2FindEqualityPayloadV2 || + if (encryptedTypeBinding == EncryptedBinDataType::kFLE2FindEqualityPayloadV2 || encryptedTypeBinding == EncryptedBinDataType::kFLE2FindRangePayloadV2) { builder->appendBinData(fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data<char>()); return; - } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayload || - encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2) { - - // TODO: SERVER-73303 set to just kFLE2InsertUpdatePayloadV2 once is enabled by default - auto validVersionedTypeBinding = - gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility) - ? EncryptedBinDataType::kFLE2InsertUpdatePayloadV2 - : EncryptedBinDataType::kFLE2InsertUpdatePayload; - uassert(7291907, - "Encountered a Queryable Encryption insert/update payload type that is no longer " - "supported", - encryptedTypeBinding == validVersionedTypeBinding); - + } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2) { uassert(6373505, "Unexpected end of iterator", it.it != it.end); const auto payload = it.it; // TODO - validate field is actually indexed in the schema? if (payload->isRangePayload()) { - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto& v2Payload = payload->payload.getInsertUpdatePayloadVersion2(); - - FLE2IndexedRangeEncryptedValueV2 sp( - v2Payload, EDCServerCollection::generateTags(*payload), payload->counts); - - uassert(7291908, - str::stream() << "Type '" << typeName(sp.bsonType) - << "' is not a valid type for Queryable Encryption Range", - isFLE2RangeIndexedSupportedType(sp.bsonType)); - - std::vector<ServerDerivedFromDataToken> edgeDerivedTokens; - auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - v2Payload.getServerEncryptionToken()); - for (auto& ets : v2Payload.getEdgeTokenSet().value()) { - edgeDerivedTokens.push_back( - FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - ets.getServerDerivedFromDataToken())); - } + auto& v2Payload = payload->payload; - auto swEncrypted = sp.serialize(serverToken, edgeDerivedTokens); - uassertStatusOK(swEncrypted); - toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2RangeIndexedValueV2, - ConstDataRange(swEncrypted.getValue()), - builder); + FLE2IndexedRangeEncryptedValueV2 sp( + v2Payload, EDCServerCollection::generateTags(*payload), payload->counts); - for (auto& mblock : sp.metadataBlocks) { - pTags->push_back({mblock.tag}); - } - it.it++; - return; - } - // TODO: SERVER-73303 delete below once is enabled by default - auto& v1Payload = payload->payload.getInsertUpdatePayloadVersion1(); - FLE2IndexedRangeEncryptedValue sp(v1Payload, payload->counts); - - uassert(6775311, + uassert(7291908, str::stream() << "Type '" << typeName(sp.bsonType) << "' is not a valid type for Queryable Encryption Range", isFLE2RangeIndexedSupportedType(sp.bsonType)); - auto swEncrypted = - sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload->payload.getServerEncryptionToken())); + std::vector<ServerDerivedFromDataToken> edgeDerivedTokens; + auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( + v2Payload.getServerEncryptionToken()); + for (auto& ets : v2Payload.getEdgeTokenSet().value()) { + edgeDerivedTokens.push_back( + FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + ets.getServerDerivedFromDataToken())); + } + + auto swEncrypted = sp.serialize(serverToken, edgeDerivedTokens); uassertStatusOK(swEncrypted); toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2RangeIndexedValue, + EncryptedBinDataType::kFLE2RangeIndexedValueV2, ConstDataRange(swEncrypted.getValue()), builder); - auto tagsRange = EDCServerCollection::generateTags(sp); - for (const auto& tag : tagsRange) { - pTags->push_back({tag}); + for (auto& mblock : sp.metadataBlocks) { + pTags->push_back({mblock.tag}); } } else { dassert(payload->counts.size() == 1); - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - auto tag = EDCServerCollection::generateTag(*payload); - auto& v2Payload = payload->payload.getInsertUpdatePayloadVersion2(); - FLE2IndexedEqualityEncryptedValueV2 sp(v2Payload, tag, payload->counts[0]); - - uassert(7291906, - str::stream() << "Type '" << typeName(sp.bsonType) - << "' is not a valid type for Queryable Encryption Equality", - isFLE2EqualityIndexedSupportedType(sp.bsonType)); - - auto swEncrypted = - sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - v2Payload.getServerEncryptionToken()), - FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - v2Payload.getServerDerivedFromDataToken())); - uassertStatusOK(swEncrypted); - toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2EqualityIndexedValueV2, - ConstDataRange(swEncrypted.getValue()), - builder); - pTags->push_back({tag}); - - it.it++; - return; - } - - // TODO: SERVER-73303 delete below once v2 is enabled by default - auto& v1Payload = payload->payload.getInsertUpdatePayloadVersion1(); - FLE2IndexedEqualityEncryptedValue sp(v1Payload, payload->counts[0]); + auto tag = EDCServerCollection::generateTag(*payload); + auto& v2Payload = payload->payload; + FLE2IndexedEqualityEncryptedValueV2 sp(v2Payload, tag, payload->counts[0]); - uassert(6373506, + uassert(7291906, str::stream() << "Type '" << typeName(sp.bsonType) << "' is not a valid type for Queryable Encryption Equality", isFLE2EqualityIndexedSupportedType(sp.bsonType)); auto swEncrypted = sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload->payload.getServerEncryptionToken())); + v2Payload.getServerEncryptionToken()), + FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + v2Payload.getServerDerivedFromDataToken())); uassertStatusOK(swEncrypted); toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2EqualityIndexedValue, + EncryptedBinDataType::kFLE2EqualityIndexedValueV2, ConstDataRange(swEncrypted.getValue()), builder); - - pTags->push_back({EDCServerCollection::generateTag(*payload)}); + pTags->push_back({tag}); } - } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValue || - encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { - auto validVersionedTypeBinding = - gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility) - ? EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2 - : EncryptedBinDataType::kFLE2UnindexedEncryptedValue; - uassert(7413902, - "Encountered a Queryable Encryption unindexed encrypted payload type that is no " - "longer supported", - encryptedTypeBinding == validVersionedTypeBinding); + } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { builder->appendBinData(fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data()); return; @@ -1927,49 +1580,10 @@ void collectIndexedFields(std::vector<EDCIndexedFields>* pFields, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); - switch (encryptedTypeBinding) { - case EncryptedBinDataType::kFLE2EqualityIndexedValue: - case EncryptedBinDataType::kFLE2RangeIndexedValue: - break; - case EncryptedBinDataType::kFLE2EqualityIndexedValueV2: - case EncryptedBinDataType::kFLE2RangeIndexedValueV2: - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled( - serverGlobalParams.featureCompatibility)) { - return; - } - break; - default: - return; - } - pFields->push_back({cdr, fieldPath.toString()}); -} - -void serializeDeletePayload(UUID keyId, FLEKeyVault* keyVault, BSONObjBuilder* builder) { - - auto indexKey = keyVault->getIndexKeyById(keyId); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverEncryptionToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - FLE2DeletePayload payload; - payload.setEcocToken(ecocToken.toCDR()); - payload.setServerEncryptionToken(serverEncryptionToken.toCDR()); - payload.serialize(builder); -} - -BSONObj getDeleteTokensBSON(const EncryptedFieldConfig& ef, FLEKeyVault* keyVault) { - BSONObjBuilder builder; - for (const auto& field : ef.getFields()) { - if (field.getQueries().has_value()) { - BSONObjBuilder subBuilder(builder.subobjStart(field.getPath())); - serializeDeletePayload(field.getKeyId(), keyVault, &subBuilder); - } + if (encryptedTypeBinding == EncryptedBinDataType::kFLE2EqualityIndexedValueV2 || + encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValueV2) { + pFields->push_back({cdr, fieldPath.toString()}); } - - return builder.obj(); } void collectFieldValidationInfo(stdx::unordered_map<std::string, ConstDataRange>* pFields, @@ -2009,19 +1623,6 @@ size_t getEstimatedTagCount(const std::vector<EDCServerPayloadInfo>& serverPaylo return total; } -// TODO: SERVER-73303 remove when v2 is enabled by default -template <typename T> -T decryptAndParseIndexedValue(ConstDataRange cdr, FLEKeyVault* keyVault) { - auto indexKeyId = uassertStatusOK(FLE2IndexedRangeEncryptedValue::readKeyId(cdr)); - - auto indexKey = keyVault->getIndexKeyById(indexKeyId); - - auto serverDataToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - - return uassertStatusOK(T::decryptAndParse(serverDataToken, cdr)); -} - /** * Return the first bit set in a integer. 1 indexed. */ @@ -2593,79 +2194,74 @@ BSONObj FLEClientCrypto::generateCompactionTokens(const EncryptedFieldConfig& cf } BSONObj FLEClientCrypto::decryptDocument(BSONObj& doc, FLEKeyVault* keyVault) { - // TODO: SERVER-73851 remove once libmongocrypt supports parsing v2 payloads - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - BSONObjBuilder builder; + // TODO: SERVER-73851 replace with commented code once mongocrypt supports v2 decryption + // auto crypt = createMongoCrypt(); - auto obj = transformBSON( - doc, [keyVault](ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { - auto [encryptedType, subCdr] = fromEncryptedConstDataRange(cdr); - if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2 || - encryptedType == EncryptedBinDataType::kFLE2RangeIndexedValueV2) { - std::vector<uint8_t> userCipherText; - BSONType type; - if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2) { - auto indexKeyId = - uassertStatusOK(FLE2IndexedEqualityEncryptedValueV2::readKeyId(subCdr)); - auto indexKey = keyVault->getIndexKeyById(indexKeyId); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( - indexKey.key); - userCipherText = uassertStatusOK( - FLE2IndexedEqualityEncryptedValueV2::parseAndDecryptCiphertext( - serverToken, subCdr)); - type = uassertStatusOK( - FLE2IndexedEqualityEncryptedValueV2::readBsonType(subCdr)); - } else { - auto indexKeyId = - uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readKeyId(subCdr)); - auto indexKey = keyVault->getIndexKeyById(indexKeyId); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( - indexKey.key); - userCipherText = uassertStatusOK( - FLE2IndexedRangeEncryptedValueV2::parseAndDecryptCiphertext(serverToken, - subCdr)); - type = - uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readBsonType(subCdr)); - } + // SymmetricKey& key = keyVault->getKMSLocalKey(); + // auto binary = MongoCryptBinary::createFromCDR(ConstDataRange(key.getKey(), + // key.getKeySize())); uassert(7132217, + // "mongocrypt_setopt_kms_provider_local failed", + // mongocrypt_setopt_kms_provider_local(crypt.get(), binary)); - auto userKeyId = uassertStatusOK(KeyIdAndValue::readKeyId(userCipherText)); - auto userKey = keyVault->getUserKeyById(userKeyId); - auto userData = - uassertStatusOK(KeyIdAndValue::decrypt(userKey.key, userCipherText)); - BSONObj obj = toBSON(type, userData); - builder->appendAs(obj.firstElement(), fieldPath); - } else if (encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { - auto [type, userData] = - FLE2UnindexedEncryptedValueV2::deserialize(keyVault, cdr); - BSONObj obj = toBSON(type, userData); - builder->appendAs(obj.firstElement(), fieldPath); - } else { - builder->appendBinData( - fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data()); - } - }); - builder.appendElements(obj); - return builder.obj(); - } + // uassert(7132218, "mongocrypt_init failed", mongocrypt_init(crypt.get())); - auto crypt = createMongoCrypt(); + // UniqueMongoCryptCtx ctx(mongocrypt_ctx_new(crypt.get())); + // auto input = MongoCryptBinary::createFromBSONObj(doc); + // mongocrypt_ctx_decrypt_init(ctx.get(), input); + // BSONObj obj = runStateMachineForDecryption(ctx.get(), keyVault); - SymmetricKey& key = keyVault->getKMSLocalKey(); - auto binary = MongoCryptBinary::createFromCDR(ConstDataRange(key.getKey(), key.getKeySize())); - uassert(7132217, - "mongocrypt_setopt_kms_provider_local failed", - mongocrypt_setopt_kms_provider_local(crypt.get(), binary)); + // return obj; - uassert(7132218, "mongocrypt_init failed", mongocrypt_init(crypt.get())); + BSONObjBuilder builder; - UniqueMongoCryptCtx ctx(mongocrypt_ctx_new(crypt.get())); - auto input = MongoCryptBinary::createFromBSONObj(doc); - mongocrypt_ctx_decrypt_init(ctx.get(), input); - BSONObj obj = runStateMachineForDecryption(ctx.get(), keyVault); + auto obj = transformBSON( + doc, [keyVault](ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { + auto [encryptedType, subCdr] = fromEncryptedConstDataRange(cdr); + if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2 || + encryptedType == EncryptedBinDataType::kFLE2RangeIndexedValueV2) { + std::vector<uint8_t> userCipherText; + BSONType type; + if (encryptedType == EncryptedBinDataType::kFLE2EqualityIndexedValueV2) { + auto indexKeyId = + uassertStatusOK(FLE2IndexedEqualityEncryptedValueV2::readKeyId(subCdr)); + auto indexKey = keyVault->getIndexKeyById(indexKeyId); + auto serverToken = + FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( + indexKey.key); + userCipherText = uassertStatusOK( + FLE2IndexedEqualityEncryptedValueV2::parseAndDecryptCiphertext(serverToken, + subCdr)); + type = + uassertStatusOK(FLE2IndexedEqualityEncryptedValueV2::readBsonType(subCdr)); + } else { + auto indexKeyId = + uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readKeyId(subCdr)); + auto indexKey = keyVault->getIndexKeyById(indexKeyId); + auto serverToken = + FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token( + indexKey.key); + userCipherText = + uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::parseAndDecryptCiphertext( + serverToken, subCdr)); + type = uassertStatusOK(FLE2IndexedRangeEncryptedValueV2::readBsonType(subCdr)); + } - return obj; + auto userKeyId = uassertStatusOK(KeyIdAndValue::readKeyId(userCipherText)); + auto userKey = keyVault->getUserKeyById(userKeyId); + auto userData = + uassertStatusOK(KeyIdAndValue::decrypt(userKey.key, userCipherText)); + BSONObj obj = toBSON(type, userData); + builder->appendAs(obj.firstElement(), fieldPath); + } else if (encryptedType == EncryptedBinDataType::kFLE2UnindexedEncryptedValueV2) { + auto [type, userData] = FLE2UnindexedEncryptedValueV2::deserialize(keyVault, cdr); + BSONObj obj = toBSON(type, userData); + builder->appendAs(obj.firstElement(), fieldPath); + } else { + builder->appendBinData(fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data()); + } + }); + builder.appendElements(obj); + return builder.obj(); } void FLEClientCrypto::validateTagsArray(const BSONObj& doc) { @@ -2703,42 +2299,6 @@ void FLEClientCrypto::validateDocument(const BSONObj& doc, auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(field.second); - // TODO: SERVER-73303 delete when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - if (configField->second.getQueries().has_value()) { - if (hasQueryType(configField->second, QueryTypeEnum::Equality)) { - uassert(6371509, - str::stream() << "Field '" << field.first - << "' is marked as equality but not indexed", - encryptedTypeBinding == - EncryptedBinDataType::kFLE2EqualityIndexedValue); - - auto ieev = decryptAndParseIndexedValue<FLE2IndexedEqualityEncryptedValue>( - subCdr, keyVault); - auto tag = EDCServerCollection::generateTag(ieev); - tags.insert({tag, field.first}); - } else if (hasQueryType(configField->second, QueryTypeEnum::RangePreview)) { - uassert(6775316, - str::stream() << "Field '" << field.first - << "' is marked as range but not indexed", - encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValue); - - auto irev = decryptAndParseIndexedValue<FLE2IndexedRangeEncryptedValue>( - subCdr, keyVault); - - auto tagsRange = EDCServerCollection::generateTags(irev); - for (const auto& tag : tagsRange) { - tags.insert({tag, field.first}); - } - } - } else { - uassert(6379105, - str::stream() << "Field '" << field.first << "' must be marked unindexed", - encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValue); - } - continue; - } - if (configField->second.getQueries().has_value()) { if (hasQueryType(configField->second, QueryTypeEnum::Equality)) { uassert(7293205, @@ -3264,38 +2824,6 @@ StatusWith<ECCNullDocument> ECCCollection::decryptNullDocument(ECCTwiceDerivedVa } -FLE2FindEqualityPayload FLEClientCrypto::serializeFindPayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t maxContentionFactor) { - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - FLE2FindEqualityPayload payload; - - payload.setEdcDerivedToken(edcDatakey.toCDR()); - payload.setEscDerivedToken(escDatakey.toCDR()); - payload.setEccDerivedToken(eccDatakey.toCDR()); - payload.setMaxCounter(maxContentionFactor); - payload.setServerEncryptionToken(serverToken.toCDR()); - - return payload; -} - FLE2FindEqualityPayloadV2 FLEClientCrypto::serializeFindPayloadV2(FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, BSONElement element, @@ -3326,54 +2854,6 @@ FLE2FindEqualityPayloadV2 FLEClientCrypto::serializeFindPayloadV2(FLEIndexKeyAnd return payload; } -FLE2FindRangePayload FLEClientCrypto::serializeFindRangePayload( - FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - const std::vector<std::string>& edges, - uint64_t maxContentionFactor, - const FLE2RangeFindSpec& spec) { - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); - - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - - // TODO - should we randomize the sort order of the edges vector? - std::vector<EdgeFindTokenSet> tokens; - for (auto const& edge : edges) { - - ConstDataRange value(edge.c_str(), edge.size()); - - EdgeFindTokenSet tokenSet; - tokenSet.setEdcDerivedToken( - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value) - .toCDR()); - - tokenSet.setEscDerivedToken( - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value) - .toCDR()); - tokenSet.setEccDerivedToken( - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value) - .toCDR()); - tokens.push_back(std::move(tokenSet)); - } - - FLE2FindRangePayload payload; - FLE2FindRangePayloadEdgesInfo edgesInfo; - - edgesInfo.setEdges(std::move(tokens)); - edgesInfo.setMaxCounter(maxContentionFactor); - edgesInfo.setServerEncryptionToken(serverToken.toCDR()); - - payload.setPayload(edgesInfo); - payload.setFirstOperator(spec.getFirstOperator()); - payload.setSecondOperator(spec.getSecondOperator()); - payload.setPayloadId(spec.getPayloadId()); - - return payload; -} FLE2FindRangePayloadV2 FLEClientCrypto::serializeFindRangePayloadV2( FLEIndexKeyAndId indexKey, @@ -3421,16 +2901,6 @@ FLE2FindRangePayloadV2 FLEClientCrypto::serializeFindRangePayloadV2( return payload; } -FLE2FindRangePayload FLEClientCrypto::serializeFindRangeStub(const FLE2RangeFindSpec& spec) { - FLE2FindRangePayload payload; - - payload.setFirstOperator(spec.getFirstOperator()); - payload.setSecondOperator(spec.getSecondOperator()); - payload.setPayloadId(spec.getPayloadId()); - - return payload; -} - FLE2FindRangePayloadV2 FLEClientCrypto::serializeFindRangeStubV2(const FLE2RangeFindSpec& spec) { FLE2FindRangePayloadV2 payload; @@ -3479,21 +2949,6 @@ BSONObj ECOCCollection::generateDocument(StringData fieldName, ConstDataRange pa return builder.obj(); } -ECOCCompactionDocument ECOCCollection::parseAndDecrypt(const BSONObj& doc, ECOCToken token) { - IDLParserContext ctx("root"); - auto ecocDoc = EcocDocument::parse(ctx, doc); - - auto swTokens = EncryptedStateCollectionTokens::decryptAndParse(token, ecocDoc.getValue()); - uassertStatusOK(swTokens); - auto& keys = swTokens.getValue(); - - ECOCCompactionDocument ret; - ret.fieldName = ecocDoc.getFieldName().toString(); - ret.esc = keys.esc; - ret.ecc = keys.ecc; - return ret; -} - ECOCCompactionDocumentV2 ECOCCollection::parseAndDecryptV2(const BSONObj& doc, ECOCToken token) { IDLParserContext ctx("root"); auto ecocDoc = EcocDocument::parse(ctx, doc); @@ -3508,173 +2963,6 @@ ECOCCompactionDocumentV2 ECOCCollection::parseAndDecryptV2(const BSONObj& doc, E return ret; } -FLE2IndexedEqualityEncryptedValue::FLE2IndexedEqualityEncryptedValue( - FLE2InsertUpdatePayload payload, uint64_t counter) - : edc(FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEdcDerivedToken())), - esc(FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEscDerivedToken())), - ecc(FLETokenFromCDR<FLETokenType::ECCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEccDerivedToken())), - count(counter), - bsonType(static_cast<BSONType>(payload.getType())), - indexKeyId(payload.getIndexKeyId()), - clientEncryptedValue(FLEUtil::vectorFromCDR(payload.getValue())) { - uassert(6373508, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload", - isValidBSONType(payload.getType())); -} - -FLE2IndexedEqualityEncryptedValue::FLE2IndexedEqualityEncryptedValue( - EDCDerivedFromDataTokenAndContentionFactorToken edcParam, - ESCDerivedFromDataTokenAndContentionFactorToken escParam, - ECCDerivedFromDataTokenAndContentionFactorToken eccParam, - uint64_t countParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> clientEncryptedValueParam) - : edc(edcParam), - esc(escParam), - ecc(eccParam), - count(countParam), - bsonType(typeParam), - indexKeyId(indexKeyIdParam), - clientEncryptedValue(clientEncryptedValueParam) {} - -StatusWith<UUID> FLE2IndexedEqualityEncryptedValue::readKeyId( - ConstDataRange serializedServerValue) { - ConstDataRangeCursor baseCdrc(serializedServerValue); - - auto swKeyId = baseCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swKeyId.isOK()) { - return {swKeyId.getStatus()}; - } - - return UUID::fromCDR(swKeyId.getValue()); -} - -StatusWith<FLE2IndexedEqualityEncryptedValue> FLE2IndexedEqualityEncryptedValue::decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { - ConstDataRangeCursor serializedServerCdrc(serializedServerValue); - - auto swIndexKeyId = serializedServerCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swIndexKeyId.isOK()) { - return {swIndexKeyId.getStatus()}; - } - - UUID indexKey = UUID::fromCDR(swIndexKeyId.getValue()); - - auto swBsonType = serializedServerCdrc.readAndAdvanceNoThrow<uint8_t>(); - if (!swBsonType.isOK()) { - return {swBsonType.getStatus()}; - } - - uassert(6373509, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload", - isValidBSONType(swBsonType.getValue())); - - auto type = static_cast<BSONType>(swBsonType.getValue()); - - auto swVec = FLEUtil::decryptData(token.toCDR(), serializedServerCdrc); - if (!swVec.isOK()) { - return swVec.getStatus(); - } - - auto data = swVec.getValue(); - - ConstDataRangeCursor serverEncryptedValueCdrc(data); - - auto swLength = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swLength.isOK()) { - return {swLength.getStatus()}; - } - - std::uint64_t length = swLength.getValue(); - - auto start = serverEncryptedValueCdrc.data(); - - auto advance = serverEncryptedValueCdrc.advanceNoThrow(length); - if (!advance.isOK()) { - return advance; - } - - std::vector<uint8_t> cipherText(length); - - std::copy(start, start + length, cipherText.data()); - - auto swCount = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swCount.isOK()) { - return {swCount.getStatus()}; - } - - auto swEdc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEdc.isOK()) { - return swEdc.getStatus(); - } - - auto swEsc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEsc.isOK()) { - return swEsc.getStatus(); - } - - auto swEcc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEcc.isOK()) { - return swEcc.getStatus(); - } - - return FLE2IndexedEqualityEncryptedValue( - EDCDerivedFromDataTokenAndContentionFactorToken(swEdc.getValue()), - ESCDerivedFromDataTokenAndContentionFactorToken(swEsc.getValue()), - ECCDerivedFromDataTokenAndContentionFactorToken(swEcc.getValue()), - swCount.getValue(), - type, - indexKey, - std::move(cipherText)); -} - - -StatusWith<std::vector<uint8_t>> FLE2IndexedEqualityEncryptedValue::serialize( - ServerDataEncryptionLevel1Token token) { - BufBuilder builder(clientEncryptedValue.size() + sizeof(uint64_t) * 2 + sizeof(PrfBlock) * 3); - - - builder.appendNum(LittleEndian<uint64_t>(clientEncryptedValue.size())); - - builder.appendBuf(clientEncryptedValue.data(), clientEncryptedValue.size()); - - builder.appendNum(LittleEndian<uint64_t>(count)); - - builder.appendStruct(edc.data); - - builder.appendStruct(esc.data); - - builder.appendStruct(ecc.data); - - dassert(builder.len() == - static_cast<int>(clientEncryptedValue.size() + sizeof(uint64_t) * 2 + - sizeof(PrfBlock) * 3)); - - auto swEncryptedData = encryptData(token.toCDR(), ConstDataRange(builder.buf(), builder.len())); - if (!swEncryptedData.isOK()) { - return swEncryptedData; - } - - auto cdrKeyId = indexKeyId.toCDR(); - auto serverEncryptedValue = swEncryptedData.getValue(); - - std::vector<uint8_t> serializedServerValue(serverEncryptedValue.size() + cdrKeyId.length() + 1); - - std::copy(cdrKeyId.data(), cdrKeyId.data() + cdrKeyId.length(), serializedServerValue.begin()); - uint8_t bsonTypeByte = bsonType; - std::copy( - &bsonTypeByte, (&bsonTypeByte) + 1, serializedServerValue.begin() + cdrKeyId.length()); - std::copy(serverEncryptedValue.begin(), - serverEncryptedValue.end(), - serializedServerValue.begin() + cdrKeyId.length() + 1); - - return serializedServerValue; -} - FLE2TagAndEncryptedMetadataBlock::FLE2TagAndEncryptedMetadataBlock(uint64_t countParam, uint64_t contentionParam, PrfBlock tagParam) @@ -3983,11 +3271,6 @@ std::vector<uint8_t> serializeUnindexedEncryptedValue(const FLEUserKeyAndId& use return buf; } -std::vector<uint8_t> FLE2UnindexedEncryptedValue::serialize(const FLEUserKeyAndId& userKey, - const BSONElement& element) { - return serializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValue>(userKey, element); -} - std::vector<uint8_t> FLE2UnindexedEncryptedValueV2::serialize(const FLEUserKeyAndId& userKey, const BSONElement& element) { return serializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValueV2>(userKey, element); @@ -4015,331 +3298,11 @@ std::pair<BSONType, std::vector<uint8_t>> deserializeUnindexedEncryptedValue(FLE return {bsonType, data}; } -std::pair<BSONType, std::vector<uint8_t>> FLE2UnindexedEncryptedValue::deserialize( - FLEKeyVault* keyVault, ConstDataRange blob) { - return deserializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValue>(keyVault, blob); -} - std::pair<BSONType, std::vector<uint8_t>> FLE2UnindexedEncryptedValueV2::deserialize( FLEKeyVault* keyVault, ConstDataRange blob) { return deserializeUnindexedEncryptedValue<FLE2UnindexedEncryptedValueV2>(keyVault, blob); } -std::vector<FLEEdgeToken> toFLEEdgeTokenSet(const FLE2InsertUpdatePayload& payload) { - const auto ets = payload.getEdgeTokenSet().get(); - std::vector<FLEEdgeToken> tokens; - tokens.reserve(ets.size()); - - for (const auto& et : ets) { - FLEEdgeToken edgeToken; - edgeToken.edc = - FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( - et.getEdcDerivedToken()); - edgeToken.esc = - FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( - et.getEscDerivedToken()); - edgeToken.ecc = - FLETokenFromCDR<FLETokenType::ECCDerivedFromDataTokenAndContentionFactorToken>( - et.getEccDerivedToken()); - - tokens.push_back(edgeToken); - } - - return tokens; -} - -FLE2IndexedRangeEncryptedValue::FLE2IndexedRangeEncryptedValue(FLE2InsertUpdatePayload payload, - std::vector<uint64_t> countersParam) - : tokens(toFLEEdgeTokenSet(payload)), - counters(std::move(countersParam)), - bsonType(static_cast<BSONType>(payload.getType())), - indexKeyId(payload.getIndexKeyId()), - clientEncryptedValue(FLEUtil::vectorFromCDR(payload.getValue())) { - uassert(6775312, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload", - isValidBSONType(payload.getType())); - uassert( - 6775313, "Mismatch between tokens and counters count", tokens.size() == counters.size()); -} - -FLE2IndexedRangeEncryptedValue::FLE2IndexedRangeEncryptedValue( - std::vector<FLEEdgeToken> tokenSet, - std::vector<uint64_t> countersParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> clientEncryptedValueParam) - : tokens(std::move(tokenSet)), - counters(countersParam), - bsonType(typeParam), - indexKeyId(indexKeyIdParam), - clientEncryptedValue(clientEncryptedValueParam) { - uassert( - 6775314, "Mismatch between tokens and counters count", tokens.size() == counters.size()); -} - -StatusWith<UUID> FLE2IndexedRangeEncryptedValue::readKeyId(ConstDataRange serializedServerValue) { - ConstDataRangeCursor baseCdrc(serializedServerValue); - - auto swKeyId = baseCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swKeyId.isOK()) { - return {swKeyId.getStatus()}; - } - - return UUID::fromCDR(swKeyId.getValue()); -} - -StatusWith<FLE2IndexedRangeEncryptedValue> FLE2IndexedRangeEncryptedValue::decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { - ConstDataRangeCursor serializedServerCdrc(serializedServerValue); - - auto swIndexKeyId = serializedServerCdrc.readAndAdvanceNoThrow<UUIDBuf>(); - if (!swIndexKeyId.isOK()) { - return {swIndexKeyId.getStatus()}; - } - - UUID indexKey = UUID::fromCDR(swIndexKeyId.getValue()); - - auto swBsonType = serializedServerCdrc.readAndAdvanceNoThrow<uint8_t>(); - if (!swBsonType.isOK()) { - return {swBsonType.getStatus()}; - } - - uassert(6775310, - "Invalid BSON Type in Queryable Encryption InsertUpdatePayload Range", - isValidBSONType(swBsonType.getValue())); - - auto type = static_cast<BSONType>(swBsonType.getValue()); - - auto swVec = FLEUtil::decryptData(token.toCDR(), serializedServerCdrc); - if (!swVec.isOK()) { - return swVec.getStatus(); - } - - auto data = swVec.getValue(); - - ConstDataRangeCursor serverEncryptedValueCdrc(data); - - auto swLength = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swLength.isOK()) { - return {swLength.getStatus()}; - } - - std::uint64_t length = swLength.getValue(); - - auto start = serverEncryptedValueCdrc.data(); - - auto advance = serverEncryptedValueCdrc.advanceNoThrow(length); - if (!advance.isOK()) { - return advance; - } - - std::vector<uint8_t> cipherText(length); - - std::copy(start, start + length, cipherText.data()); - - auto swEdgeCount = - serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint32_t>>(); - if (!swEdgeCount.isOK()) { - return {swEdgeCount.getStatus()}; - } - - uassert(6775315, "Edge count must not be over 129", swEdgeCount.getValue() <= 129); - - std::vector<FLEEdgeToken> tokens; - tokens.reserve(swEdgeCount.getValue()); - - std::vector<uint64_t> counters; - counters.reserve(swEdgeCount.getValue()); - - for (size_t i = 0; i < swEdgeCount.getValue(); i++) { - - auto swEdc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEdc.isOK()) { - return swEdc.getStatus(); - } - - auto swEsc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEsc.isOK()) { - return swEsc.getStatus(); - } - - auto swEcc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); - if (!swEcc.isOK()) { - return swEcc.getStatus(); - } - tokens.push_back( - FLEEdgeToken{EDCDerivedFromDataTokenAndContentionFactorToken(swEdc.getValue()), - ESCDerivedFromDataTokenAndContentionFactorToken(swEsc.getValue()), - ECCDerivedFromDataTokenAndContentionFactorToken(swEcc.getValue())}); - - auto swCount = - serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); - if (!swCount.isOK()) { - return {swCount.getStatus()}; - } - counters.push_back(swCount.getValue()); - } - - return FLE2IndexedRangeEncryptedValue(tokens, counters, type, indexKey, std::move(cipherText)); -} - - -StatusWith<std::vector<uint8_t>> FLE2IndexedRangeEncryptedValue::serialize( - ServerDataEncryptionLevel1Token token) { - BufBuilder builder(clientEncryptedValue.size() + sizeof(uint64_t) * 1 + - sizeof(uint64_t) * counters.size() + sizeof(uint32_t) + - tokens.size() * sizeof(PrfBlock) * 3); - - - builder.appendNum(LittleEndian<uint64_t>(clientEncryptedValue.size())); - - builder.appendBuf(clientEncryptedValue.data(), clientEncryptedValue.size()); - - builder.appendNum(LittleEndian<uint32_t>(tokens.size())); - - size_t c = 0; - for (auto const& ets : tokens) { - - builder.appendStruct(ets.edc.data); - - builder.appendStruct(ets.esc.data); - - builder.appendStruct(ets.ecc.data); - - builder.appendNum(LittleEndian<uint64_t>(counters[c++])); - } - - dassert(builder.len() == - static_cast<int>(clientEncryptedValue.size() + sizeof(uint64_t) * 1 + - sizeof(uint64_t) * counters.size() + sizeof(uint32_t) + - sizeof(PrfBlock) * 3 * tokens.size())); - - auto swEncryptedData = encryptData(token.toCDR(), ConstDataRange(builder.buf(), builder.len())); - if (!swEncryptedData.isOK()) { - return swEncryptedData; - } - - auto cdrKeyId = indexKeyId.toCDR(); - auto serverEncryptedValue = swEncryptedData.getValue(); - - std::vector<uint8_t> serializedServerValue(serverEncryptedValue.size() + cdrKeyId.length() + 1); - - std::copy(cdrKeyId.data(), cdrKeyId.data() + cdrKeyId.length(), serializedServerValue.begin()); - uint8_t bsonTypeByte = bsonType; - std::copy( - &bsonTypeByte, (&bsonTypeByte) + 1, serializedServerValue.begin() + cdrKeyId.length()); - std::copy(serverEncryptedValue.begin(), - serverEncryptedValue.end(), - serializedServerValue.begin() + cdrKeyId.length() + 1); - - return serializedServerValue; -} - -VersionedEdgeTokenSet::VersionedEdgeTokenSet(EdgeTokenSet ets) : edgeTokenSet(std::move(ets)) {} - -VersionedEdgeTokenSet::VersionedEdgeTokenSet(EdgeTokenSetV2 ets) : edgeTokenSet(std::move(ets)) {} - -ConstDataRange VersionedEdgeTokenSet::getEscDerivedToken() const { - return stdx::visit( - OverloadedVisitor{[](const EdgeTokenSet& ets) { return ets.getEscDerivedToken(); }, - [](const EdgeTokenSetV2& ets) { - return ets.getEscDerivedToken(); - }}, - edgeTokenSet); -} - -ConstDataRange VersionedEdgeTokenSet::getEncryptedTokens() const { - return stdx::visit( - OverloadedVisitor{[](const EdgeTokenSet& ets) { return ets.getEncryptedTokens(); }, - [](const EdgeTokenSetV2& ets) { - return ets.getEncryptedTokens(); - }}, - edgeTokenSet); -} - -VersionedInsertUpdatePayload::VersionedInsertUpdatePayload(FLE2InsertUpdatePayload iup) - : iupayload(std::move(iup)), edgeTokenSet(convertPayloadEdgeTokenSet<decltype(iup)>()) {} - -VersionedInsertUpdatePayload::VersionedInsertUpdatePayload(FLE2InsertUpdatePayloadV2 iup) - : iupayload(std::move(iup)), edgeTokenSet(convertPayloadEdgeTokenSet<decltype(iup)>()) {} - -const FLE2InsertUpdatePayload& VersionedInsertUpdatePayload::getInsertUpdatePayloadVersion1() - const { - auto payloadPtr = stdx::get_if<FLE2InsertUpdatePayload>(&iupayload); - uassert( - 7291904, "Attempted to retrieve invalid version of FLE2InsertUpdatePayload", payloadPtr); - return *payloadPtr; -} - -const FLE2InsertUpdatePayloadV2& VersionedInsertUpdatePayload::getInsertUpdatePayloadVersion2() - const { - auto payloadPtr = stdx::get_if<FLE2InsertUpdatePayloadV2>(&iupayload); - uassert( - 7291905, "Attempted to retrieve invalid version of FLE2InsertUpdatePayload", payloadPtr); - return *payloadPtr; -} - -const mongo::UUID& VersionedInsertUpdatePayload::getIndexKeyId() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) -> const mongo::UUID& { - return v1.getIndexKeyId(); - }, - [](const FLE2InsertUpdatePayloadV2& v2) -> const mongo::UUID& { - return v2.getIndexKeyId(); - }}, - iupayload); -} - -int VersionedInsertUpdatePayload::getType() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getType(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getType(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getEncryptedTokens() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getEncryptedTokens(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getEncryptedTokens(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getEscDerivedToken() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getEscDerivedToken(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getEscDerivedToken(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getEdcDerivedToken() const { - return stdx::visit( - OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { return v1.getEdcDerivedToken(); }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getEdcDerivedToken(); - }}, - iupayload); -} - -ConstDataRange VersionedInsertUpdatePayload::getServerEncryptionToken() const { - return stdx::visit(OverloadedVisitor{[](const FLE2InsertUpdatePayload& v1) { - return v1.getServerEncryptionToken(); - }, - [](const FLE2InsertUpdatePayloadV2& v2) { - return v2.getServerEncryptionToken(); - }}, - iupayload); -} - -const boost::optional<std::vector<VersionedEdgeTokenSet>>& -VersionedInsertUpdatePayload::getEdgeTokenSet() const { - return edgeTokenSet; -} FLE2IndexedRangeEncryptedValueV2::FLE2IndexedRangeEncryptedValueV2( const FLE2InsertUpdatePayloadV2& payload, @@ -4451,11 +3414,8 @@ FLE2IndexedRangeEncryptedValueV2::parseAndValidateFields(ConstDataRange serializ sizeof(FLE2TagAndEncryptedMetadataBlock::SerializedBlob))); } - return {{UUID::fromCDR(swIndexKeyId.getValue()), - type, - edgeCount, - encryptedDataCdrc, - metadataBlocks}}; + return FLE2IndexedRangeEncryptedValueV2::ParsedFields{ + UUID::fromCDR(swIndexKeyId.getValue()), type, edgeCount, encryptedDataCdrc, metadataBlocks}; } StatusWith<std::vector<uint8_t>> FLE2IndexedRangeEncryptedValueV2::parseAndDecryptCiphertext( @@ -4606,13 +3566,8 @@ void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, visitEncryptedBSON(obj, [&indexedFields](ConstDataRange cdr, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); - auto expectedPayloadType = EncryptedBinDataType::kFLE2InsertUpdatePayload; - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - expectedPayloadType = EncryptedBinDataType::kFLE2InsertUpdatePayloadV2; - } - - if (encryptedTypeBinding == expectedPayloadType) { + if (encryptedTypeBinding == EncryptedBinDataType::kFLE2InsertUpdatePayloadV2) { uassert(6373601, str::stream() << "Field '" << fieldPath << "' is encrypted, but absent from schema", @@ -4627,12 +3582,6 @@ void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, } void EDCServerCollection::validateModifiedDocumentCompatibility(BSONObj& obj) { - - // TODO: SERVER-73303 delete when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - return; - } - visitEncryptedBSON(obj, [](ConstDataRange cdr, StringData fieldPath) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); switch (encryptedTypeBinding) { @@ -4683,40 +3632,9 @@ PrfBlock EDCServerCollection::generateTag(const EDCServerPayloadInfo& payload) { return generateTag(edcTwiceDerived, payload.counts[0]); } -// TODO: SERVER-73303 delete when v2 is enabled by default -PrfBlock EDCServerCollection::generateTag(const FLE2IndexedEqualityEncryptedValue& indexedValue) { - auto edcTwiceDerived = - FLETwiceDerivedTokenGenerator::generateEDCTwiceDerivedToken(indexedValue.edc); - return generateTag(edcTwiceDerived, indexedValue.count); -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -PrfBlock EDCServerCollection::generateTag(const FLEEdgeToken& token, FLECounter count) { - auto edcTwiceDerived = FLETwiceDerivedTokenGenerator::generateEDCTwiceDerivedToken(token.edc); - return generateTag(edcTwiceDerived, count); -} - -// TODO: SERVER-73303 delete when v2 is enabled by default -std::vector<PrfBlock> EDCServerCollection::generateTags( - const FLE2IndexedRangeEncryptedValue& indexedValue) { - uassert(6775317, - "Mismatch between tokens and counters count", - indexedValue.tokens.size() == indexedValue.counters.size()); - - std::vector<PrfBlock> tags; - tags.reserve(indexedValue.tokens.size()); - - for (size_t i = 0; i < indexedValue.tokens.size(); i++) { - tags.push_back( - EDCServerCollection::generateTag(indexedValue.tokens[i], indexedValue.counters[i])); - } - - return tags; -} - std::vector<PrfBlock> EDCServerCollection::generateTags(const EDCServerPayloadInfo& rangePayload) { // throws if EDCServerPayloadInfo has invalid payload version - auto& v2Payload = rangePayload.payload.getInsertUpdatePayloadVersion2(); + auto& v2Payload = rangePayload.payload; uassert(7291909, "InsertUpdatePayload must have an edge token set", @@ -4738,30 +3656,6 @@ std::vector<PrfBlock> EDCServerCollection::generateTags(const EDCServerPayloadIn return tags; } -StatusWith<FLE2IndexedEqualityEncryptedValue> EDCServerCollection::decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { - auto pair = fromEncryptedConstDataRange(serializedServerValue); - uassert(6672412, - "Wrong encrypted field type", - pair.first == EncryptedBinDataType::kFLE2EqualityIndexedValue); - - return FLE2IndexedEqualityEncryptedValue::decryptAndParse(token, pair.second); -} - -StatusWith<FLE2IndexedEqualityEncryptedValue> EDCServerCollection::decryptAndParse( - ConstDataRange token, ConstDataRange serializedServerValue) { - auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>(token); - - return FLE2IndexedEqualityEncryptedValue::decryptAndParse(serverToken, serializedServerValue); -} - -StatusWith<FLE2IndexedRangeEncryptedValue> EDCServerCollection::decryptAndParseRange( - ConstDataRange token, ConstDataRange serializedServerValue) { - auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>(token); - - return FLE2IndexedRangeEncryptedValue::decryptAndParse(serverToken, serializedServerValue); -} - std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> EDCServerCollection::generateEDCTokens( EDCDerivedFromDataToken token, uint64_t maxContentionFactor) { std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> tokens; @@ -4905,62 +3799,6 @@ BSONObj EDCServerCollection::finalizeForUpdate( return builder.obj(); } -BSONObj EDCServerCollection::generateUpdateToRemoveTags( - const std::vector<EDCIndexedFields>& removedFields, const StringMap<FLEDeleteToken>& tokenMap) { - std::vector<TagInfo> tags; - - for (const auto& field : removedFields) { - auto tokenIt = tokenMap.find(field.fieldPathName); - uassert(6371513, - str::stream() << "Could not find field'" << field.fieldPathName - << "' in delete token map.", - tokenIt != tokenMap.end()); - auto token = *tokenIt; - - auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(field.value); - auto localEncryptedTypeBinding = - encryptedTypeBinding; // Workaround the fact that we cannot pass a structured binding - // variable into a lambda - uassert(6371514, - str::stream() << "Field'" << field.fieldPathName - << "' in not a supported encrypted type: " - << EncryptedBinDataType_serializer(localEncryptedTypeBinding), - encryptedTypeBinding == EncryptedBinDataType::kFLE2EqualityIndexedValue || - encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValue); - - if (encryptedTypeBinding == EncryptedBinDataType::kFLE2RangeIndexedValue) { - auto range = uassertStatusOK(FLE2IndexedRangeEncryptedValue::decryptAndParse( - token.second.serverEncryptionToken, subCdr)); - - for (size_t i = 0; i < range.counters.size(); i++) { - auto tag = EDCServerCollection::generateTag(range.tokens[i], range.counters[i]); - tags.push_back({tag}); - } - } else { - auto ieev = uassertStatusOK(FLE2IndexedEqualityEncryptedValue::decryptAndParse( - token.second.serverEncryptionToken, subCdr)); - auto tag = EDCServerCollection::generateTag(ieev); - - tags.push_back({tag}); - } - } - - // Build { $pull : {__safeContent__ : {$in : [tag..] } } } - BSONObjBuilder builder; - { - BSONObjBuilder subBuilder(builder.subobjStart(kDollarPull)); - BSONObjBuilder pushBuilder(subBuilder.subobjStart(kSafeContent)); - BSONArrayBuilder arrayBuilder(pushBuilder.subarrayStart(kDollarIn)); - - // Add new tags - for (auto const& tag : tags) { - appendTag(tag.tag, &arrayBuilder); - } - } - - return builder.obj(); -} - BSONObj EDCServerCollection::generateUpdateToRemoveTags(const std::vector<PrfBlock>& tagsToPull) { uassert(7293203, "Cannot generate update command to remove tags with empty tags", @@ -5063,10 +3901,6 @@ EncryptedFieldConfig EncryptionInformationHelpers::getAndValidateSchema( auto efc = EncryptedFieldConfig::parse(IDLParserContext("schema"), element.Obj()); - // TODO: SERVER-73303 remove when v2 is enabled by default - if (!gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(6371206, "Expected a value for eccCollection", efc.getEccCollection().has_value()); - } uassert(6371207, "Expected a value for escCollection", efc.getEscCollection().has_value()); uassert(6371208, "Expected a value for ecocCollection", efc.getEcocCollection().has_value()); @@ -5083,48 +3917,6 @@ std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(Cons return {subType, cdrc}; } -BSONObj EncryptionInformationHelpers::encryptionInformationSerializeForDelete( - const NamespaceString& nss, const EncryptedFieldConfig& ef, FLEKeyVault* keyVault) { - - EncryptionInformation ei; - ei.setType(kEncryptionInformationSchemaVersion); - - ei.setSchema(BSON(nss.toString() << ef.toBSON())); - - ei.setDeleteTokens(BSON(nss.toString() << getDeleteTokensBSON(ef, keyVault))); - - return ei.toBSON(); -} - -StringMap<FLEDeleteToken> EncryptionInformationHelpers::getDeleteTokens( - const NamespaceString& nss, const EncryptionInformation& ei) { - uassert(6371308, "DeleteTokens is empty", ei.getDeleteTokens().has_value()); - - BSONObj deleteTokens = ei.getDeleteTokens().value(); - - auto element = deleteTokens.getField(nss.toString()); - - uassert(6371309, - "DeleteTokens item for namespace is not set", - !element.eoo() && element.type() == Object); - - StringMap<FLEDeleteToken> map; - for (const auto& deleteTokenBSON : element.Obj()) { - uassert(6371310, "DeleteToken is not an object", deleteTokenBSON.type() == Object); - - auto payload = FLE2DeletePayload::parse(IDLParserContext("delete"), deleteTokenBSON.Obj()); - - auto deleteToken = - FLEDeleteToken{FLETokenFromCDR<FLETokenType::ECOCToken>(payload.getEcocToken()), - FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload.getServerEncryptionToken())}; - - map.insert({deleteTokenBSON.fieldNameStringData().toString(), deleteToken}); - } - - return map; -} - ParsedFindEqualityPayload::ParsedFindEqualityPayload(BSONElement fleFindPayload) : ParsedFindEqualityPayload(binDataToCDR(fleFindPayload)){}; @@ -5135,41 +3927,17 @@ ParsedFindEqualityPayload::ParsedFindEqualityPayload(ConstDataRange cdr) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); auto encryptedType = encryptedTypeBinding; - // TODO: SERVER-73303 refactor when v2 is enabled by default - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7292600, - str::stream() << "Unexpected encrypted payload type: " - << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindEqualityPayloadV2); - - auto payload = parseFromCDR<FLE2FindEqualityPayloadV2>(subCdr); - - escToken = - FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(payload.getEscDerivedToken()); - edcToken = - FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(payload.getEdcDerivedToken()); - serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - payload.getServerDerivedFromDataToken()); - maxCounter = payload.getMaxCounter(); - return; - } - - uassert(6435600, + uassert(7292600, str::stream() << "Unexpected encrypted payload type: " << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindEqualityPayload); + encryptedType == EncryptedBinDataType::kFLE2FindEqualityPayloadV2); - auto payload = parseFromCDR<FLE2FindEqualityPayload>(subCdr); + auto payload = parseFromCDR<FLE2FindEqualityPayloadV2>(subCdr); escToken = FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(payload.getEscDerivedToken()); - eccToken = FLETokenFromCDR<FLETokenType::ECCDerivedFromDataToken>(payload.getEccDerivedToken()); edcToken = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(payload.getEdcDerivedToken()); - - if (payload.getServerEncryptionToken().has_value()) { - serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload.getServerEncryptionToken().value()); - } - + serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + payload.getServerDerivedFromDataToken()); maxCounter = payload.getMaxCounter(); } @@ -5183,46 +3951,12 @@ ParsedFindRangePayload::ParsedFindRangePayload(ConstDataRange cdr) { auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); auto encryptedType = encryptedTypeBinding; - // TODO: SERVER-73303 refactor when v2 is enabled by default - if (gFeatureFlagFLE2ProtocolVersion2.isEnabled(serverGlobalParams.featureCompatibility)) { - uassert(7292601, - str::stream() << "Unexpected encrypted payload type: " - << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindRangePayloadV2); - - auto payload = parseFromCDR<FLE2FindRangePayloadV2>(subCdr); - payloadId = payload.getPayloadId(); - firstOp = payload.getFirstOperator(); - secondOp = payload.getSecondOperator(); - - if (!payload.getPayload()) { - return; - } - - edges = std::vector<FLEFindEdgeTokenSet>(); - auto& edgesRef = edges.value(); - auto& info = payload.getPayload().value(); - - for (auto const& edge : info.getEdges()) { - auto escToken = - FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(edge.getEscDerivedToken()); - auto edcToken = - FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(edge.getEdcDerivedToken()); - auto serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( - edge.getServerDerivedFromDataToken()); - edgesRef.push_back({edcToken, escToken, {}, serverDataDerivedToken}); - } - - maxCounter = info.getMaxCounter(); - return; - } - - uassert(6869501, + uassert(7292601, str::stream() << "Unexpected encrypted payload type: " << static_cast<uint32_t>(encryptedType), - encryptedType == EncryptedBinDataType::kFLE2FindRangePayload); + encryptedType == EncryptedBinDataType::kFLE2FindRangePayloadV2); - auto payload = parseFromCDR<FLE2FindRangePayload>(subCdr); + auto payload = parseFromCDR<FLE2FindRangePayloadV2>(subCdr); payloadId = payload.getPayloadId(); firstOp = payload.getFirstOperator(); secondOp = payload.getSecondOperator(); @@ -5233,23 +3967,18 @@ ParsedFindRangePayload::ParsedFindRangePayload(ConstDataRange cdr) { edges = std::vector<FLEFindEdgeTokenSet>(); auto& edgesRef = edges.value(); - auto& info = payload.getPayload().value(); for (auto const& edge : info.getEdges()) { - auto escToken = FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(edge.getEscDerivedToken()); - auto eccToken = - FLETokenFromCDR<FLETokenType::ECCDerivedFromDataToken>(edge.getEccDerivedToken()); auto edcToken = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(edge.getEdcDerivedToken()); - edgesRef.push_back({edcToken, escToken, eccToken, {}}); + auto serverDataDerivedToken = FLETokenFromCDR<FLETokenType::ServerDerivedFromDataToken>( + edge.getServerDerivedFromDataToken()); + edgesRef.push_back({edcToken, escToken, serverDataDerivedToken}); } - serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - info.getServerEncryptionToken()); - maxCounter = info.getMaxCounter(); } @@ -5284,29 +4013,6 @@ void CompactionHelpers::validateCompactionTokens(const EncryptedFieldConfig& efc } } -std::vector<ECCDocument> CompactionHelpers::mergeECCDocuments(std::vector<ECCDocument>& unmerged) { - std::vector<ECCDocument> merged; - std::sort(unmerged.begin(), unmerged.end()); - - for (size_t i = 0; i < unmerged.size();) { - merged.push_back(unmerged[i]); - auto& last = merged.back(); - i++; - for (; i < unmerged.size() && ((last.end + 1) == unmerged[i].start); i++) { - last.end = unmerged[i].end; - } - } - return merged; -} - -uint64_t CompactionHelpers::countDeleted(const std::vector<ECCDocument>& rangeList) { - uint64_t sum = 0; - for (auto& range : rangeList) { - sum += range.end - range.start + 1; - } - return sum; -} - ConstDataRange binDataToCDR(BSONElement element) { uassert(6338501, "Expected binData BSON element", element.type() == BinData); @@ -5808,48 +4514,6 @@ OSTType_Decimal128 getTypeInfoDecimal128(Decimal128 value, return {mapping, 0, std::numeric_limits<boost::multiprecision::uint128_t>::max()}; } - -EncryptedPredicateEvaluator::EncryptedPredicateEvaluator(ConstDataRange serverToken, - int64_t contentionFactor, - std::vector<ConstDataRange> edcTokens) - : _serverToken(PrfBlockfromCDR(serverToken)), _contentionFactor(contentionFactor) { - for (auto cdr : edcTokens) { - _edcTokens.push_back(PrfBlockfromCDR(cdr)); - } - for (auto& prf : _edcTokens) { - for (auto& token : - EDCServerCollection::generateEDCTokens(ConstDataRange(prf), _contentionFactor)) { - _cachedEDCTokens.insert(std::move(token.data)); - } - } -} - -bool EncryptedPredicateEvaluator::evaluate( - Value fieldValue, - EncryptedBinDataType indexedValueType, - std::function<std::vector<EDCDerivedFromDataTokenAndContentionFactorToken>( - ConstDataRange, ConstDataRange)> decryptAndParse) const { - - if (fieldValue.getType() != BinData) { - return false; - } - - auto [subSubType, data] = fromEncryptedBinData(fieldValue); - - uassert(6672400, "Invalid encrypted indexed field", subSubType == indexedValueType); - - // Value matches if - // 1. Decrypt field is successful - // 2. EDC_u Token is in GenTokens(EDC Token, ContentionFactor) - // - auto tokens = decryptAndParse(ConstDataRange(_serverToken), data); - - return std::any_of( - std::make_move_iterator(tokens.begin()), - std::make_move_iterator(tokens.end()), - [this](auto&& token) { return _cachedEDCTokens.count(std::move(token.data)) == 1; }); -} - EncryptedPredicateEvaluatorV2::EncryptedPredicateEvaluatorV2( std::vector<ServerZerosEncryptionToken> zerosTokens) : _zerosDecryptionTokens(std::move(zerosTokens)){}; diff --git a/src/mongo/crypto/fle_crypto.h b/src/mongo/crypto/fle_crypto.h index 03636e902bc..7b49241fd54 100644 --- a/src/mongo/crypto/fle_crypto.h +++ b/src/mongo/crypto/fle_crypto.h @@ -750,20 +750,6 @@ using ContentionFactorFn = std::function<uint64_t(const FLE2EncryptionPlaceholde class FLEClientCrypto { public: - // TODO: SERVER-73303 delete v1 serialize methods when v2 is enabled by default - static FLE2FindEqualityPayload serializeFindPayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - BSONElement element, - uint64_t maxContentionFactor); - - static FLE2FindRangePayload serializeFindRangePayload(FLEIndexKeyAndId indexKey, - FLEUserKeyAndId userKey, - const std::vector<std::string>& edges, - uint64_t maxContentionFactor, - const FLE2RangeFindSpec& spec); - - static FLE2FindRangePayload serializeFindRangeStub(const FLE2RangeFindSpec& spec); - static FLE2FindEqualityPayloadV2 serializeFindPayloadV2(FLEIndexKeyAndId indexKey, FLEUserKeyAndId userKey, BSONElement element, @@ -881,24 +867,6 @@ public: ESCDerivedFromDataTokenAndContentionFactorToken esc; }; -// TODO: SERVER-73303 delete when v2 is enabled by default -struct ECOCCompactionDocument { - - bool operator==(const ECOCCompactionDocument& other) const { - return (fieldName == other.fieldName) && (esc == other.esc) && (ecc == other.ecc); - } - - template <typename H> - friend H AbslHashValue(H h, const ECOCCompactionDocument& doc) { - return H::combine(std::move(h), doc.fieldName, doc.esc, doc.ecc); - } - - // Id is not included as it unimportant - std::string fieldName; - ESCDerivedFromDataTokenAndContentionFactorToken esc; - ECCDerivedFromDataTokenAndContentionFactorToken ecc; -}; - struct ECOCCompactionDocumentV2 { bool operator==(const ECOCCompactionDocumentV2& other) const { return (fieldName == other.fieldName) && (esc == other.esc); @@ -933,68 +901,9 @@ class ECOCCollection { public: static BSONObj generateDocument(StringData fieldName, ConstDataRange payload); - // TODO: SERVER-73303 delete when v2 is enabled by default - static ECOCCompactionDocument parseAndDecrypt(const BSONObj& doc, ECOCToken token); - static ECOCCompactionDocumentV2 parseAndDecryptV2(const BSONObj& doc, ECOCToken token); }; - -/** - * Class to read/write FLE2 Equality Indexed Encrypted Values - * - * Fields are encrypted with the following: - * - * struct { - * uint8_t fle_blob_subtype = 7; - * uint8_t key_uuid[16]; - * uint8_t original_bson_type; - * ciphertext[ciphertext_length]; - * } - * - * Encrypt(ServerDataEncryptionLevel1Token, Struct(K_KeyId, v, count, d, s, c)) - * - * struct { - * uint64_t length; - * uint8_t[length] cipherText; // UserKeyId + Encrypt(K_KeyId, value), - * uint64_t counter; - * uint8_t[32] edc; // EDCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] esc; // ESCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] ecc; // ECCDerivedFromDataTokenAndContentionFactorToken - *} - * - * The specification needs to be in sync with the validation in 'bson_validate.cpp'. - */ -struct FLE2IndexedEqualityEncryptedValue { - FLE2IndexedEqualityEncryptedValue(FLE2InsertUpdatePayload payload, uint64_t counter); - - FLE2IndexedEqualityEncryptedValue(EDCDerivedFromDataTokenAndContentionFactorToken edcParam, - ESCDerivedFromDataTokenAndContentionFactorToken escParam, - ECCDerivedFromDataTokenAndContentionFactorToken eccParam, - uint64_t countParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> serializedServerValueParam); - - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); - - /** - * Read the key id from the payload. - */ - static StatusWith<UUID> readKeyId(ConstDataRange serializedServerValue); - - StatusWith<std::vector<uint8_t>> serialize(ServerDataEncryptionLevel1Token token); - - EDCDerivedFromDataTokenAndContentionFactorToken edc; - ESCDerivedFromDataTokenAndContentionFactorToken esc; - ECCDerivedFromDataTokenAndContentionFactorToken ecc; - uint64_t count; - BSONType bsonType; - UUID indexKeyId; - std::vector<uint8_t> clientEncryptedValue; -}; - /** * Class to read/write the metadata block consisting of the encrypted counter * and contention factor, the tag, and the encrypted 128-bit string of zeros. @@ -1119,32 +1028,6 @@ struct FLE2IndexedEqualityEncryptedValueV2 { FLE2TagAndEncryptedMetadataBlock metadataBlock; }; -// TODO: SERVER-73303 delete when v2 is enabled by default -/** - * Class to read/write FLE2 Unindexed Encrypted Values - * - * Fields are encrypted with the following: - * - * struct { - * uint8_t fle_blob_subtype = 6; - * uint8_t key_uuid[16]; - * uint8_t original_bson_type; - * ciphertext[ciphertext_length]; - * } blob; - * - * The specification needs to be in sync with the validation in 'bson_validate.cpp'. - */ -struct FLE2UnindexedEncryptedValue { - static std::vector<uint8_t> serialize(const FLEUserKeyAndId& userKey, - const BSONElement& element); - static std::pair<BSONType, std::vector<uint8_t>> deserialize(FLEKeyVault* keyVault, - ConstDataRange blob); - - static constexpr crypto::aesMode mode = crypto::aesMode::ctr; - static constexpr EncryptedBinDataType fleType = - EncryptedBinDataType::kFLE2UnindexedEncryptedValue; - static constexpr size_t assocDataSize = sizeof(uint8_t) + sizeof(UUID) + sizeof(uint8_t); -}; /** * Class to read/write FLE2 Unindexed Encrypted Values (for protocol version 2) @@ -1191,62 +1074,6 @@ struct FLEEdgeToken { }; /** - * Class to read/write FLE2 Range Indexed Encrypted Values - * - * Fields are encrypted with the following: - * - * struct { - * uint8_t fle_blob_subtype = 9; - * uint8_t key_uuid[16]; - * uint8_t original_bson_type; - * ciphertext[ciphertext_length]; - * } - * - * Encrypt(ServerDataEncryptionLevel1Token, Struct(K_KeyId, v, edgeCount, [count, d, s, c] x - *edgeCount )) - * - * struct { - * uint64_t length; - * uint8_t[length] cipherText; // UserKeyId + Encrypt(K_KeyId, value), - * uint32_t edgeCount; - * struct { - * uint64_t counter; - * uint8_t[32] edc; // EDCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] esc; // ESCDerivedFromDataTokenAndContentionFactorToken - * uint8_t[32] ecc; // ECCDerivedFromDataTokenAndContentionFactorToken - * } edges[edgeCount]; - *} - * - * The specification needs to be in sync with the validation in 'bson_validate.cpp'. - */ -struct FLE2IndexedRangeEncryptedValue { - FLE2IndexedRangeEncryptedValue(FLE2InsertUpdatePayload payload, - std::vector<uint64_t> countersParam); - - FLE2IndexedRangeEncryptedValue(std::vector<FLEEdgeToken> tokens, - std::vector<uint64_t> countersParam, - BSONType typeParam, - UUID indexKeyIdParam, - std::vector<uint8_t> serializedServerValueParam); - - static StatusWith<FLE2IndexedRangeEncryptedValue> decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); - - /** - * Read the key id from the payload. - */ - static StatusWith<UUID> readKeyId(ConstDataRange serializedServerValue); - - StatusWith<std::vector<uint8_t>> serialize(ServerDataEncryptionLevel1Token token); - - std::vector<FLEEdgeToken> tokens; - std::vector<uint64_t> counters; - BSONType bsonType; - UUID indexKeyId; - std::vector<uint8_t> clientEncryptedValue; -}; - -/** * Class to read/write QE protocol version 2 of Range Indexed * Encrypted Values. * @@ -1310,62 +1137,6 @@ struct FLE2IndexedRangeEncryptedValueV2 { std::vector<FLE2TagAndEncryptedMetadataBlock> metadataBlocks; }; -// TODO: SERVER-73303 delete when v2 is enabled by default -/* - * Shim layer for EdgeTokenSet types with different protocol versions. - */ -class VersionedEdgeTokenSet { -public: - VersionedEdgeTokenSet() = default; - VersionedEdgeTokenSet(EdgeTokenSet ets); - VersionedEdgeTokenSet(EdgeTokenSetV2 ets); - - ConstDataRange getEscDerivedToken() const; - ConstDataRange getEncryptedTokens() const; - -private: - stdx::variant<EdgeTokenSet, EdgeTokenSetV2> edgeTokenSet; -}; - -// TODO: SERVER-73303 delete when v2 is enabled by default -/* - * Shim layer for FLE2InsertUpdatePayload types with different protocol versions. - */ -class VersionedInsertUpdatePayload { -public: - VersionedInsertUpdatePayload() = default; - VersionedInsertUpdatePayload(FLE2InsertUpdatePayload iup); - VersionedInsertUpdatePayload(FLE2InsertUpdatePayloadV2 iup); - - const FLE2InsertUpdatePayload& getInsertUpdatePayloadVersion1() const; - const FLE2InsertUpdatePayloadV2& getInsertUpdatePayloadVersion2() const; - - const mongo::UUID& getIndexKeyId() const; - int getType() const; - ConstDataRange getEncryptedTokens() const; - ConstDataRange getEscDerivedToken() const; - ConstDataRange getEdcDerivedToken() const; - ConstDataRange getServerEncryptionToken() const; - const boost::optional<std::vector<VersionedEdgeTokenSet>>& getEdgeTokenSet() const; - -private: - template <class Payload> - boost::optional<std::vector<VersionedEdgeTokenSet>> convertPayloadEdgeTokenSet() { - boost::optional<std::vector<VersionedEdgeTokenSet>> converted; - auto& payload = stdx::get<Payload>(iupayload); - if (payload.getEdgeTokenSet().has_value()) { - auto& etsList = payload.getEdgeTokenSet().value(); - converted = std::vector<VersionedEdgeTokenSet>(etsList.size()); - std::transform(etsList.begin(), etsList.end(), edgeTokenSet->begin(), [](auto& ets) { - return VersionedEdgeTokenSet(ets); - }); - } - return converted; - } - stdx::variant<FLE2InsertUpdatePayload, FLE2InsertUpdatePayloadV2> iupayload; - boost::optional<std::vector<VersionedEdgeTokenSet>> edgeTokenSet; -}; - struct EDCServerPayloadInfo { static ESCDerivedFromDataTokenAndContentionFactorToken getESCToken(ConstDataRange cdr); @@ -1373,8 +1144,7 @@ struct EDCServerPayloadInfo { return payload.getEdgeTokenSet().has_value(); } - // TODO: SERVER-73303 change type to FLE2InsertUpdatePayloadV2 when v2 is enabled by default - VersionedInsertUpdatePayload payload; + FLE2InsertUpdatePayloadV2 payload; std::string fieldPathName; std::vector<uint64_t> counts; }; @@ -1429,22 +1199,14 @@ public: * Used during updates to verify that the modified document's pre-image can be * safely updated per the protocol compatibility rules. */ - static void validateModifiedDocumentCompatibility(BSONObj& originalDocument); + static void validateModifiedDocumentCompatibility(BSONObj& obj); + /** * Get information about all FLE2InsertUpdatePayload payloads */ static std::vector<EDCServerPayloadInfo> getEncryptedFieldInfo(BSONObj& obj); - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); - - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ConstDataRange token, ConstDataRange serializedServerValue); - - static StatusWith<FLE2IndexedRangeEncryptedValue> decryptAndParseRange( - ConstDataRange token, ConstDataRange serializedServerValue); - /** * Generate a search tag * @@ -1452,9 +1214,7 @@ public: */ static PrfBlock generateTag(EDCTwiceDerivedToken edcTwiceDerived, FLECounter count); static PrfBlock generateTag(const EDCServerPayloadInfo& payload); - static PrfBlock generateTag(const FLE2IndexedEqualityEncryptedValue& indexedValue); static PrfBlock generateTag(const FLEEdgeToken& token, FLECounter count); - static std::vector<PrfBlock> generateTags(const FLE2IndexedRangeEncryptedValue& indexedValue); static std::vector<PrfBlock> generateTags(const EDCServerPayloadInfo& rangePayload); /** @@ -1483,17 +1243,6 @@ public: static BSONObj finalizeForUpdate(const BSONObj& doc, const std::vector<EDCServerPayloadInfo>& serverPayload); - // TODO: SERVER-73303 remove once v2 is enabled by default - /** - * Generate an update modifier document with $pull to remove stale tags. - * - * Generates: - * - * { $pull : {__safeContent__ : {$in : [tag..] } } } - */ - static BSONObj generateUpdateToRemoveTags(const std::vector<EDCIndexedFields>& removedFields, - const StringMap<FLEDeleteToken>& tokenMap); - /** * Generate an update modifier document with $pull to remove stale tags. * @@ -1549,27 +1298,11 @@ public: static BSONObj encryptionInformationSerialize(const NamespaceString& nss, const BSONObj& encryptedFields); - // TODO: SERVER-73303 remove when v2 is enabled by default - /** - * Serialize EncryptionInformation with EncryptionInformation.schema and a map of delete tokens - * for each field in EncryptedFieldConfig. - */ - static BSONObj encryptionInformationSerializeForDelete(const NamespaceString& nss, - const EncryptedFieldConfig& ef, - FLEKeyVault* keyVault); - /** * Get a schema from EncryptionInformation and ensure the esc/ecc/ecoc are setup correctly. */ static EncryptedFieldConfig getAndValidateSchema(const NamespaceString& nss, const EncryptionInformation& ei); - - // TODO: SERVER-73303 remove when v2 is enabled by default - /** - * Get a set of delete tokens for a given nss from EncryptionInformation. - */ - static StringMap<FLEDeleteToken> getDeleteTokens(const NamespaceString& nss, - const EncryptionInformation& ei); }; /** @@ -1595,25 +1328,6 @@ public: * in the encrypted field config */ static void validateCompactionTokens(const EncryptedFieldConfig& efc, BSONObj compactionTokens); - - // TODO: SERVER-73303 delete when v2 is enabled by default - /** - * Merges the list of ECCDocuments so that entries whose tuple values are - * adjacent to each other are combined into a single entry. For example, - * the input [ (1,3), (11,11), (7,9), (4,6) ] outputs [ (1,9), (11,11) ]. - * Assumes none of the input entries overlap with each other. - * - * This will sort the input unmerged list as a side-effect. - */ - static std::vector<ECCDocument> mergeECCDocuments(std::vector<ECCDocument>& unmerged); - - // TODO: SERVER-73303 delete when v2 is enabled by default - /** - * Given a list of ECCDocument, where each document is a range of - * deleted positions, this calculates the total number of deleted - * positions. - */ - static uint64_t countDeleted(const std::vector<ECCDocument>& rangeList); }; /** @@ -1626,10 +1340,7 @@ std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(Cons struct ParsedFindEqualityPayload { ESCDerivedFromDataToken escToken; - // TODO: SERVER-73303 remove eccToken and serverToken when v2 is enabled by default - ECCDerivedFromDataToken eccToken; EDCDerivedFromDataToken edcToken; - boost::optional<ServerDataEncryptionLevel1Token> serverToken; boost::optional<std::int64_t> maxCounter; // v2 fields @@ -1726,18 +1437,12 @@ struct FLEFindEdgeTokenSet { EDCDerivedFromDataToken edc; ESCDerivedFromDataToken esc; - // TODO: SERVER-73303 remove ecc field when v2 is enabled by default - ECCDerivedFromDataToken ecc; - ServerDerivedFromDataToken server; }; struct ParsedFindRangePayload { boost::optional<std::vector<FLEFindEdgeTokenSet>> edges; - // TODO: SERVER-73303 remove serverToken when v2 is enabled by default - ServerDataEncryptionLevel1Token serverToken; - Fle2RangeOperator firstOp; boost::optional<Fle2RangeOperator> secondOp; std::int32_t payloadId{}; diff --git a/src/mongo/crypto/fle_crypto_predicate.h b/src/mongo/crypto/fle_crypto_predicate.h index be4856edb72..79f6c2037cc 100644 --- a/src/mongo/crypto/fle_crypto_predicate.h +++ b/src/mongo/crypto/fle_crypto_predicate.h @@ -51,45 +51,6 @@ namespace mongo { -// TODO: SERVER-73303 delete class when v2 is enabled by default -class EncryptedPredicateEvaluator { -public: - EncryptedPredicateEvaluator(ConstDataRange serverToken, - int64_t contentionFactor, - std::vector<ConstDataRange> edcTokens); - - EncryptedPredicateEvaluator() {} - - /** - * Evaluates an encrypted predicate (equality or range), as defined by a contention factor along - * with a list of edc tokens, over an encrypted value. - * - * Returns a boolean indicator. - */ - bool evaluate(Value fieldValue, - EncryptedBinDataType indexedValueType, - std::function<std::vector<EDCDerivedFromDataTokenAndContentionFactorToken>( - ConstDataRange, ConstDataRange)> decryptAndParse) const; - - std::vector<PrfBlock> edcTokens() const { - return _edcTokens; - } - - PrfBlock serverToken() const { - return _serverToken; - } - - int64_t contentionFactor() const { - return _contentionFactor; - } - -private: - PrfBlock _serverToken; - std::vector<PrfBlock> _edcTokens; - int64_t _contentionFactor{0}; - stdx::unordered_set<PrfBlock> _cachedEDCTokens; -}; - class EncryptedPredicateEvaluatorV2 { public: EncryptedPredicateEvaluatorV2(std::vector<ServerZerosEncryptionToken> zerosTokens); diff --git a/src/mongo/crypto/fle_crypto_test.cpp b/src/mongo/crypto/fle_crypto_test.cpp index 8514057d678..15acde90f69 100644 --- a/src/mongo/crypto/fle_crypto_test.cpp +++ b/src/mongo/crypto/fle_crypto_test.cpp @@ -365,16 +365,6 @@ TEST(FLETokens, TestVectorUnindexedValueDecryption) { // Unindexed field decryption // Encryption can not be generated using test vectors because IV is random TestKeyVault keyVault; - { - const std::string uxCiphertext = hexblob::decode( - "06ABCDEFAB12349876123412345678901202F2CE7FDD0DECD5442CC98C10B9138741785173E323132982740496768877A3BA46581CED4A34031B1174B5C524C15BAAE687F88C29FC71F40A32BCD53D63CDA0A6646E8677E167BB3A933529F5B519CFE255BBC323D943B4F105"_sd); - auto [uxBsonType, uxPlaintext] = - FLE2UnindexedEncryptedValue::deserialize(&keyVault, ConstDataRange(uxCiphertext)); - ASSERT_EQUALS(uxBsonType, BSONType::String); - ASSERT_EQUALS( - hexblob::encode(uxPlaintext.data(), uxPlaintext.size()), - "260000004C6F7279207761732061206D6F75736520696E2061206269672062726F776E20686F75736500"); - } { const std::string uxCiphertext = hexblob::decode( @@ -388,36 +378,6 @@ TEST(FLETokens, TestVectorUnindexedValueDecryption) { } } -TEST(FLETokens, TestVectorIndexedValueDecryption) { - // Equality indexed field decryption - // Encryption can not be generated using test vectors because IV is random - - const std::string ixCiphertext = hexblob::decode( - "000000000000000000000000000000000297044B8E1B5CF4F9052EDB50236A343597C418A74352F98357A77E0D4299C04151CBEC24A5D5349A5A5EAA1FE334154FEEB6C8E7BD636089904F76950B2184D146792CBDF9179FFEDDB7D90FC257BB13DCB3E731182A447E2EF1BE7A2AF13DC9362701BABDE0B5E78CF4A92227D5B5D1E1556E75BAB5B4E9F5CEFEA3BA3E3D5D31D11B20619437A30550EFF5B602357567CF05058E4F84A103293F70302F3A50667642DD0325D194A197"_sd); - ServerDataEncryptionLevel1Token serverEncryptToken( - decodePrf("EB9A73F7912D86A4297E81D2F675AF742874E4057E3A890FEC651A23EEE3F3EC"_sd)); - - auto swServerPayload = FLE2IndexedEqualityEncryptedValue::decryptAndParse( - serverEncryptToken, ConstDataRange(ixCiphertext)); - ASSERT_OK(swServerPayload.getStatus()); - - auto cdrEqualHex = [](ConstDataRange cdr, const StringData hex) -> bool { - const std::string s = hexblob::decode(hex); - return cdr.length() == s.size() && std::equal(s.begin(), s.end(), cdr.data<char>()); - }; - - auto sp = swServerPayload.getValue(); - ASSERT(cdrEqualHex(sp.edc.toCDR(), - "97C8DFE394D80A4EE335E3F9FDC024D18BE4B92F9444FCA316FF9896D7BF455D"_sd)); - ASSERT(cdrEqualHex(sp.esc.toCDR(), - "EBB22F74BE0FA4AD863188D3F33AF0B95CB4CA4ED0091E1A43513DB20E9D59AE"_sd)); - ASSERT(cdrEqualHex(sp.ecc.toCDR(), - "A1DF0BB04C977BD4BC0B487FFFD2E3BBB96078354DE9F204EE5872BB10F01971"_sd)); - ASSERT_EQ(sp.count, 123456); - ASSERT(cdrEqualHex( - sp.clientEncryptedValue, - "260000004C6F7279207761732061206D6F75736520696E2061206269672062726F776E20686F75736500"_sd)); -} TEST(FLETokens, TestVectorESCCollectionDecryptDocument) { ESCTwiceDerivedTagToken escTwiceTag( @@ -1969,163 +1929,6 @@ TEST(FLE_EDC, ServerSide_Payloads_V2_IsValidZerosBlob) { ASSERT_FALSE(FLE2TagAndEncryptedMetadataBlock::isValidZerosBlob(zeros)); } -TEST(FLE_EDC, ServerSide_Payloads) { - TestKeyVault keyVault; - - auto doc = BSON("sample" << 123456); - auto element = doc.firstElement(); - - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(getIndexKey()); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(getIndexKey()); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - FLECounter counter = 0; - - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, counter); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, counter); - - FLE2InsertUpdatePayload iupayload; - - - iupayload.setEdcDerivedToken(edcDatakey.toCDR()); - iupayload.setEscDerivedToken(escDatakey.toCDR()); - iupayload.setEccDerivedToken(eccDatakey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - iupayload.setValue(value); - iupayload.setType(element.type()); - - FLE2IndexedEqualityEncryptedValue serverPayload(iupayload, 123456); - - auto swBuf = serverPayload.serialize(serverEncryptToken); - ASSERT_OK(swBuf.getStatus()); - - auto swServerPayload = - FLE2IndexedEqualityEncryptedValue::decryptAndParse(serverEncryptToken, swBuf.getValue()); - - ASSERT_OK(swServerPayload.getStatus()); - auto sp = swServerPayload.getValue(); - ASSERT_EQ(sp.edc, serverPayload.edc); - ASSERT_EQ(sp.esc, serverPayload.esc); - ASSERT_EQ(sp.ecc, serverPayload.ecc); - ASSERT_EQ(sp.count, serverPayload.count); - ASSERT(sp.clientEncryptedValue == serverPayload.clientEncryptedValue); - ASSERT_EQ(serverPayload.clientEncryptedValue.size(), value.length()); - ASSERT(std::equal(serverPayload.clientEncryptedValue.begin(), - serverPayload.clientEncryptedValue.end(), - value.data<uint8_t>())); -} - -TEST(FLE_EDC, ServerSide_Range_Payloads) { - TestKeyVault keyVault; - - auto doc = BSON("sample" << 3); - auto element = doc.firstElement(); - - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(getIndexKey()); - auto serverEncryptToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(getIndexKey()); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); - auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); - auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); - - FLECounter counter = 0; - - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - ESCDerivedFromDataToken escDatakey = - FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); - ECCDerivedFromDataToken eccDatakey = - FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); - - - ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, counter); - ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = - FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, counter); - - FLE2InsertUpdatePayload iupayload; - - - iupayload.setEdcDerivedToken(edcDatakey.toCDR()); - iupayload.setEscDerivedToken(escDatakey.toCDR()); - iupayload.setEccDerivedToken(eccDatakey.toCDR()); - iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); - - auto swEncryptedTokens = - EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); - uassertStatusOK(swEncryptedTokens); - iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); - iupayload.setValue(value); - iupayload.setType(element.type()); - - std::vector<EdgeTokenSet> tokens; - EdgeTokenSet ets; - ets.setEdcDerivedToken(edcDatakey.toCDR()); - ets.setEscDerivedToken(escDatakey.toCDR()); - ets.setEccDerivedToken(eccDatakey.toCDR()); - ets.setEncryptedTokens(swEncryptedTokens.getValue()); - - tokens.push_back(ets); - tokens.push_back(ets); - - iupayload.setEdgeTokenSet(tokens); - - FLE2IndexedRangeEncryptedValue serverPayload(iupayload, {123456, 123456}); - - auto swBuf = serverPayload.serialize(serverEncryptToken); - ASSERT_OK(swBuf.getStatus()); - - auto swServerPayload = - FLE2IndexedRangeEncryptedValue::decryptAndParse(serverEncryptToken, swBuf.getValue()); - - ASSERT_OK(swServerPayload.getStatus()); - auto sp = swServerPayload.getValue(); - ASSERT_EQ(sp.tokens.size(), 2); - for (size_t i = 0; i < sp.tokens.size(); i++) { - auto ets = sp.tokens[i]; - auto rhs = serverPayload.tokens[i]; - ASSERT_EQ(ets.edc, rhs.edc); - ASSERT_EQ(ets.esc, rhs.esc); - ASSERT_EQ(ets.ecc, rhs.ecc); - ASSERT_EQ(sp.counters[i], serverPayload.counters[i]); - } - - ASSERT(sp.clientEncryptedValue == serverPayload.clientEncryptedValue); - ASSERT_EQ(serverPayload.clientEncryptedValue.size(), value.length()); - ASSERT(std::equal(serverPayload.clientEncryptedValue.begin(), - serverPayload.clientEncryptedValue.end(), - value.data<uint8_t>())); -} TEST(FLE_EDC, ServerSide_Range_Payloads_V2) { TestKeyVault keyVault; @@ -2411,7 +2214,6 @@ TEST(EncryptionInformation, BadSchema) { } TEST(EncryptionInformation, MissingStateCollection) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); { @@ -2509,58 +2311,6 @@ TEST(IndexedFields, DuplicateIndexKeyIds) { ASSERT_THROWS_CODE(encryptDocument(builder.obj(), &keyVault), DBException, 6371407); } -TEST(DeleteTokens, Basic) { - TestKeyVault keyVault; - NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); - EncryptedFieldConfig efc = getTestEncryptedFieldConfig(); - - auto obj = - EncryptionInformationHelpers::encryptionInformationSerializeForDelete(ns, efc, &keyVault); - - std::cout << "Tokens" << obj << std::endl; -} - -TEST(DeleteTokens, Fetch) { - TestKeyVault keyVault; - NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); - EncryptedFieldConfig efc = getTestEncryptedFieldConfig(); - - auto obj = - EncryptionInformationHelpers::encryptionInformationSerializeForDelete(ns, efc, &keyVault); - - auto tokenMap = EncryptionInformationHelpers::getDeleteTokens( - ns, EncryptionInformation::parse(IDLParserContext("foo"), obj)); - - ASSERT_EQ(tokenMap.size(), 2); - - ASSERT(tokenMap.contains("nested.encrypted")); - ASSERT(tokenMap.contains("encrypted")); -} - -TEST(DeleteTokens, CorruptDelete) { - TestKeyVault keyVault; - NamespaceString ns = NamespaceString::createNamespaceString_forTest("test.test"); - EncryptedFieldConfig efc = getTestEncryptedFieldConfig(); - - EncryptionInformation ei; - ei.setType(1); - - ei.setSchema(BSON(ns.toString() << efc.toBSON())); - - // Missing Delete tokens - ASSERT_THROWS_CODE(EncryptionInformationHelpers::getDeleteTokens(ns, ei), DBException, 6371308); - - // NSS map is not an object - ei.setDeleteTokens(BSON(ns.toString() << "str")); - - ASSERT_THROWS_CODE(EncryptionInformationHelpers::getDeleteTokens(ns, ei), DBException, 6371309); - - // Tokens is not a map - ei.setDeleteTokens(BSON(ns.toString() << BSON("a" - << "b"))); - - ASSERT_THROWS_CODE(EncryptionInformationHelpers::getDeleteTokens(ns, ei), DBException, 6371310); -} // Verify we can compare two list of tags correctly TEST(TagDelta, Basic) { @@ -2650,20 +2400,6 @@ TEST(EDC, UnindexedEncryptDecrypt) { std::vector<uint8_t>(element.value(), element.value() + element.valuesize()); { - auto blob = FLE2UnindexedEncryptedValue::serialize(userKey, element); - ASSERT_EQ(blob[0], 6); - - // assert length of ciphertext (including HMAC & IV) is consistent with CTR mode - auto cipherTextLen = blob.size() - FLE2UnindexedEncryptedValue::assocDataSize; - ASSERT_EQ(cipherTextLen, - crypto::fle2AeadCipherOutputLength(elementData.size(), crypto::aesMode::ctr)); - - auto [type, plainText] = FLE2UnindexedEncryptedValue::deserialize(&keyVault, {blob}); - ASSERT_EQ(type, element.type()); - ASSERT_TRUE( - std::equal(plainText.begin(), plainText.end(), elementData.begin(), elementData.end())); - } - { auto blob = FLE2UnindexedEncryptedValueV2::serialize(userKey, element); ASSERT_EQ(blob[0], 16); @@ -3124,43 +2860,6 @@ std::vector<ECCDocument> pairsToECCDocuments( return output; } -TEST(CompactionHelpersTest, mergeECCDocumentsTest) { - std::vector<ECCDocument> input, output, expected; - - // Test empty input - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output.empty()); - - // Test single pair - input = pairsToECCDocuments({{15, 20}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output == input); - - // Test input with no gaps - input = pairsToECCDocuments({{15, 20}, {13, 13}, {1, 6}, {7, 12}, {14, 14}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT_EQ(output.size(), 1); - ASSERT_EQ(output.front().start, 1); - ASSERT_EQ(output.front().end, 20); - - // Test input with gaps; nothing is merged - input = pairsToECCDocuments({{5, 5}, {12, 16}, {9, 9}, {23, 45}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output == input); - - // Test input with gaps; at least one merged - input = pairsToECCDocuments({{5, 5}, {12, 16}, {6, 9}, {17, 23}, {45, 45}}); - expected = pairsToECCDocuments({{5, 9}, {12, 23}, {45, 45}}); - output = CompactionHelpers::mergeECCDocuments(input); - ASSERT(output == expected); -} - -TEST(CompactionHelpersTest, countDeletedTest) { - ASSERT_EQ(CompactionHelpers::countDeleted({}), 0); - - auto input = pairsToECCDocuments({{15, 20}, {13, 13}, {1, 6}, {7, 12}, {14, 14}}); - ASSERT_EQ(CompactionHelpers::countDeleted(input), 20); -} TEST(EDCServerCollectionTest, GenerateEDCTokens) { @@ -3183,8 +2882,6 @@ TEST(EDCServerCollectionTest, GenerateEDCTokens) { } TEST(EDCServerCollectionTest, ValidateModifiedDocumentCompatibility) { - RAIIServerParameterControllerForTest controller("featureFlagFLE2ProtocolVersion2", true); - std::vector<uint8_t> blob; std::vector<EncryptedBinDataType> badTypes = { EncryptedBinDataType::kFLE2EqualityIndexedValue, diff --git a/src/mongo/crypto/fle_field_schema.idl b/src/mongo/crypto/fle_field_schema.idl index 8e1d03c6cc8..d8a6f1fdfaf 100644 --- a/src/mongo/crypto/fle_field_schema.idl +++ b/src/mongo/crypto/fle_field_schema.idl @@ -187,27 +187,6 @@ structs: optional: true validator: { gte: 1, lte: 4 } - EdgeTokenSet: - description: "Payload of an indexed field to insert or update" - strict: true - fields: - d: - description: "EDCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: eccDerivedToken - p: - description: "Encrypted tokens" - type: bindata_generic - cpp_name: encryptedTokens - EdgeTokenSetV2: description: "Payload of an indexed field to insert or update. Version 2" strict: true @@ -229,48 +208,6 @@ structs: type: bindata_generic cpp_name: encryptedTokens - FLE2InsertUpdatePayload: - description: "Payload of an indexed field to insert or update" - strict: true - fields: - d: - description: "EDCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataTokenAndCounter" - type: bindata_generic - cpp_name: eccDerivedToken - p: - description: "Encrypted tokens" - type: bindata_generic - cpp_name: encryptedTokens - u: - description: "Index KeyId" - type: uuid - cpp_name: indexKeyId - t: - description: "Encrypted type" - type: safeInt - cpp_name: type - v: - description: "Encrypted value" - type: bindata_generic - cpp_name: value - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - g: - description: "Array of Edges" - type: array<EdgeTokenSet> - cpp_name: edgeTokenSet - optional: true - FLE2InsertUpdatePayloadV2: description: "Payload of an indexed field to insert or update. Version 2" strict: true @@ -318,46 +255,6 @@ structs: cpp_name: edgeTokenSet optional: true - FLE2DeletePayload: - description: "Payload of an indexed field to delete" - strict: true - fields: - o: - description: "ECOCToken" - type: bindata_generic - cpp_name: ecocToken - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - - FLE2FindEqualityPayload: - description: "Payload for an equality find" - strict: true - fields: - d: - description: "EDCDerivedFromDataToken" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataToken" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataToken" - type: bindata_generic - cpp_name: eccDerivedToken - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - optional: true # For backwards compat, make this optional - cm: - description: "Queryable Encryption max counter" - type: long - cpp_name: maxCounter - optional: true - FLE2FindEqualityPayloadV2: description: "Payload for an equality find. Version 2" strict: true @@ -380,23 +277,6 @@ structs: cpp_name: maxCounter optional: true - EdgeFindTokenSet: - description: "Payload of an edge to query for" - strict: true - fields: - d: - description: "EDCDerivedFromDataToken" - type: bindata_generic - cpp_name: edcDerivedToken - s: - description: "ESCDerivedFromDataToken" - type: bindata_generic - cpp_name: escDerivedToken - c: - description: "ECCDerivedFromDataToken" - type: bindata_generic - cpp_name: eccDerivedToken - EdgeFindTokenSetV2: description: "Payload of an edge to query for. Version 2" strict: true @@ -414,23 +294,6 @@ structs: type: bindata_generic cpp_name: serverDerivedFromDataToken - FLE2FindRangePayloadEdgesInfo: - description: "Token information for find range payload." - strict: true - fields: - g: - description: "Array of Edges" - type: array<EdgeFindTokenSet> - cpp_name: edges - e: - description: "ServerDataEncryptionLevel1Token" - type: bindata_generic - cpp_name: serverEncryptionToken - cm: - description: "Queryable Encryption max counter" - type: long - cpp_name: maxCounter - FLE2FindRangePayloadEdgesInfoV2: description: "Token information for find range payload." strict: true @@ -444,27 +307,6 @@ structs: type: long cpp_name: maxCounter - FLE2FindRangePayload: - description: "Payload for a range find" - strict: true - fields: - payload: - description: "Token information for a find range payload" - type: FLE2FindRangePayloadEdgesInfo - optional: true - payloadId: - description: "Id of payload - must be paired with another payload" - type: safeInt - optional: false - firstOperator: - description: "First query operator for which this payload was generated." - type: Fle2RangeOperator - optional: false - secondOperator: - description: "Second query operator for which this payload was generated. Only populated for two-sided ranges." - type: Fle2RangeOperator - optional: true - FLE2FindRangePayloadV2: description: "Payload for a range find. Version 2" strict: true diff --git a/src/mongo/crypto/fle_stats.h b/src/mongo/crypto/fle_stats.h index 6b94c2427a9..39eec428097 100644 --- a/src/mongo/crypto/fle_stats.h +++ b/src/mongo/crypto/fle_stats.h @@ -101,13 +101,6 @@ public: _hasStats.store(true); accumulateStats(_compactStats.getEsc(), stats.getEsc()); accumulateStats(_compactStats.getEcoc(), stats.getEcoc()); - - if (stats.getEcc().has_value()) { - if (!_compactStats.getEcc().has_value()) { - _compactStats.setEcc(ECStats{}); - } - accumulateStats(_compactStats.getEcc().value(), stats.getEcc().value()); - } } private: diff --git a/src/mongo/crypto/fle_stats.idl b/src/mongo/crypto/fle_stats.idl index fdd6026657a..485a59f8f50 100644 --- a/src/mongo/crypto/fle_stats.idl +++ b/src/mongo/crypto/fle_stats.idl @@ -60,10 +60,10 @@ structs: default: 0 CompactStats: - description: "Stats about records in ECC, ECOC, and ESC compact touched" + description: "Stats about records in ECOC, and ESC compact touched" fields: ecoc: ECOCStats - ecc: # TODO: SERVER-73303 remove ecc when v2 is enabled by default + ecc: # TODO: SERVER-68373 remove ecc when 7.0 becomes last LTS type: ECStats optional: true esc: ECStats diff --git a/src/mongo/crypto/fle_tags.cpp b/src/mongo/crypto/fle_tags.cpp index c1c07e391b8..ffd901e3567 100644 --- a/src/mongo/crypto/fle_tags.cpp +++ b/src/mongo/crypto/fle_tags.cpp @@ -95,123 +95,6 @@ size_t sizeArrayElementsMemory(size_t tagCount) { return size; } - -// TODO: SERVER-73303 remove when v2 is enabled by default -// The algorithm for constructing a list of tags matching an equality predicate on an encrypted -// field is as follows: -// -// (1) Query ESC to obtain the counter value (n) after the most recent insert. -// (2) Query ECC for a null document. -// (a) A null document means there has been at least one compaction of ECC. -// (b) No null document means there has not been a compaction. Therefore we have to check all -// the tags from 1..n to see if they have been deleted. -// (3) Return the surviving set of tags from 1..n (encrypted). -// -// Given: -// s: ESCDerivedFromDataToken -// c: ECCDerivedFromDataToken -// d: EDCDerivedFromDataToken -// Do: -// n = ESC::emuBinary(s) -// deletedTags = [] -// pos = ECC::nullDocument(c) ? ECC::nullDocument(c).position : 1 -// while true { -// if (doc = ECC::getDocument(c, pos); doc != null) { -// deletedTags = [doc | deletedTags] -// } else { -// break -// } -// pos++ -// } -// return [EDC::encrypt(i) | i in [1..n] where i not in deletedTags] -std::vector<PrfBlock> readTagsWithContention(const FLEStateCollectionReader& esc, - const FLEStateCollectionReader& ecc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - uint64_t cf, - size_t memoryLimit, - std::vector<PrfBlock>&& binaryTags) { - - auto escTok = DerivedToken::generateESCDerivedFromDataTokenAndContentionFactorToken(s, cf); - auto escTag = TwiceDerived::generateESCTwiceDerivedTagToken(escTok); - auto escVal = TwiceDerived::generateESCTwiceDerivedValueToken(escTok); - - auto eccTok = DerivedToken::generateECCDerivedFromDataTokenAndContentionFactorToken(c, cf); - auto eccTag = TwiceDerived::generateECCTwiceDerivedTagToken(eccTok); - auto eccVal = TwiceDerived::generateECCTwiceDerivedValueToken(eccTok); - - auto edcTok = DerivedToken::generateEDCDerivedFromDataTokenAndContentionFactorToken(d, cf); - auto edcTag = TwiceDerived::generateEDCTwiceDerivedToken(edcTok); - - // (1) Query ESC for the counter value after the most recent insert. - // 0 => 0 inserts for this field value pair. - // n => n inserts for this field value pair. - // none => compaction => query ESC for null document to find # of inserts. - auto insertCounter = ESCCollection::emuBinary(esc, escTag, escVal); - if (insertCounter && insertCounter.value() == 0) { - return std::move(binaryTags); - } - - auto numInserts = insertCounter - ? uassertStatusOK( - ESCCollection::decryptDocument( - escVal, esc.getById(ESCCollection::generateId(escTag, insertCounter)))) - .count - : uassertStatusOK(ESCCollection::decryptNullDocument( - escVal, esc.getById(ESCCollection::generateId(escTag, boost::none)))) - .count; - - // (2) Query ECC for a null document. - auto eccNullDoc = ecc.getById(ECCCollection::generateId(eccTag, boost::none)); - auto pos = eccNullDoc.isEmpty() - ? 1 - : uassertStatusOK(ECCCollection::decryptNullDocument(eccVal, eccNullDoc)).position + 2; - - std::vector<ECCDocument> deletes; - - // (2) Search ECC for deleted tag(counter) values. - while (true) { - auto eccObj = ecc.getById(ECCCollection::generateId(eccTag, pos)); - if (eccObj.isEmpty()) { - break; - } - auto eccDoc = uassertStatusOK(ECCCollection::decryptDocument(eccVal, eccObj)); - // Compaction placeholders only present for positive contention factors (cm). - if (eccDoc.valueType == ECCValueType::kCompactionPlaceholder) { - break; - } - // Note, in the worst case where no compactions have occurred, the deletes vector will grow - // proportionally to the number of deletes. This is not likely to present a significant - // problem but we should still track the memory we consume here. - deletes.emplace_back(eccDoc); - pos++; - } - - std::sort(deletes.begin(), deletes.end()); - - auto numDeletes = std::accumulate(deletes.begin(), deletes.end(), 0, [](auto acc, auto eccDoc) { - return acc + eccDoc.end - eccDoc.start + 1; - }); - auto cumTagCount = binaryTags.size() + numInserts - numDeletes; - - verifyTagsWillFit(cumTagCount, memoryLimit); - - for (uint64_t i = 1; i <= numInserts; i++) { - if (auto it = std::lower_bound( - deletes.begin(), - deletes.end(), - i, - [](ECCDocument& eccDoc, uint64_t tag) { return eccDoc.end < tag; }); - it != deletes.end() && it->start <= i && i <= it->end) { - continue; - } - // (3) Return the surviving set of tags from 1..n (encrypted). - binaryTags.emplace_back(EDCServerCollection::generateTag(edcTag, i)); - } - return std::move(binaryTags); -} - // A positive contention factor (cm) means we must run the above algorithm (cm) times. std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc, @@ -268,37 +151,4 @@ std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, return binaryTags; } - -// TODO: SERVER-73303 remove when v2 is enabled by default -// A positive contention factor (cm) means we must run the above algorithm (cm) times. -std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - boost::optional<int64_t> cm) { - - // The output of readTags will be used as the argument to a $in expression, so make sure we - // don't exceed the configured memory limit. - auto memoryLimit = static_cast<size_t>(internalQueryFLERewriteMemoryLimit.load()); - auto contentionMax = cm.value_or(0); - std::vector<PrfBlock> binaryTags; - - auto makeCollectionReader = [](FLETagQueryInterface* queryImpl, const NamespaceString& nss) { - auto docCount = queryImpl->countDocuments(nss); - return TxnCollectionReader(docCount, queryImpl, nss); - }; - - // Construct FLE rewriter from the transaction client and encryptionInformation. - auto esc = makeCollectionReader(queryImpl, nssEsc); - auto ecc = makeCollectionReader(queryImpl, nssEcc); - - for (auto i = 0; i <= contentionMax; i++) { - binaryTags = - readTagsWithContention(esc, ecc, s, c, d, i, memoryLimit, std::move(binaryTags)); - } - - return binaryTags; -} } // namespace mongo::fle diff --git a/src/mongo/crypto/fle_tags.h b/src/mongo/crypto/fle_tags.h index 9597924b6ea..ac69de0350a 100644 --- a/src/mongo/crypto/fle_tags.h +++ b/src/mongo/crypto/fle_tags.h @@ -40,32 +40,9 @@ class FLETagQueryInterface; namespace mongo::fle { /** - * Read a list of binary tags given ESC, ECC, and EDC derived tokens and a specific contention + * Read a list of binary tags given ESC and and EDC derived tokens and a maximum contention * factor. */ -std::vector<PrfBlock> readTagsWithContention(const FLEStateCollectionReader& esc, - const FLEStateCollectionReader& ecc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - uint64_t contentionFactor, - size_t memoryLimit, - std::vector<PrfBlock>&& binaryTags); - -/** - * Read a list of binary tags given ESC, ECC, and EDC derived tokens and a maximum contention - * factor. - * - * TODO: SERVER-73303 remove when v2 is enabled by default - */ -std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, - const NamespaceString& nssEsc, - const NamespaceString& nssEcc, - ESCDerivedFromDataToken s, - ECCDerivedFromDataToken c, - EDCDerivedFromDataToken d, - boost::optional<int64_t> cm); - std::vector<PrfBlock> readTags(FLETagQueryInterface* queryImpl, const NamespaceString& nssEsc, ESCDerivedFromDataToken s, |