diff options
author | Mark Benvenuto <mark.benvenuto@mongodb.com> | 2022-02-22 13:48:40 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-02-23 03:39:31 +0000 |
commit | a49e1ce0dedf13fbf87a4430a98a7f6407ea5b57 (patch) | |
tree | 16e82dc728032723b2eb809e2e57f619fddffa15 /src/mongo | |
parent | 54b918d73c63483faecedb807722a707ba0e64c2 (diff) | |
download | mongo-a49e1ce0dedf13fbf87a4430a98a7f6407ea5b57.tar.gz |
SERVER-63735 Add support for equality indexed encryption
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/crypto/fle_crypto.cpp | 498 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto.h | 117 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto_test.cpp | 423 | ||||
-rw-r--r-- | src/mongo/rpc/rewrite_state_change_errors.cpp | 2 |
4 files changed, 1032 insertions, 8 deletions
diff --git a/src/mongo/crypto/fle_crypto.cpp b/src/mongo/crypto/fle_crypto.cpp index 19220953fbc..b24d9ca01b3 100644 --- a/src/mongo/crypto/fle_crypto.cpp +++ b/src/mongo/crypto/fle_crypto.cpp @@ -121,13 +121,20 @@ PrfBlock blockToArray(SHA256Block& block) { return data; } +PrfBlock PrfBlockfromCDR(ConstDataRange block) { + uassert(6373501, "Invalid prf length", block.length() == sizeof(PrfBlock)); + + PrfBlock ret; + std::copy(block.data(), block.data() + block.length(), ret.data()); + return ret; +} + PrfBlock prf(ConstDataRange key, ConstDataRange cdr) { SHA256Block block; SHA256Block::computeHmac(key.data<uint8_t>(), key.length(), {cdr}, &block); return blockToArray(block); } - PrfBlock prf(ConstDataRange key, uint64_t value) { std::array<char, sizeof(uint64_t)> bufValue; DataView(bufValue.data()).write<LittleEndian<uint64_t>>(value); @@ -182,6 +189,10 @@ void toBinData(StringData field, std::vector<uint8_t>& block, BSONObjBuilder* bu builder->appendBinData(field, block.size(), BinDataType::BinDataGeneral, block.data()); } +void appendTag(PrfBlock block, BSONArrayBuilder* builder) { + builder->appendBinData(block.size(), BinDataType::BinDataGeneral, block.data()); +} + template <typename T> T parseFromCDR(ConstDataRange cdr) { ConstDataRangeCursor cdc(cdr); @@ -191,6 +202,11 @@ T parseFromCDR(ConstDataRange cdr) { return T::parse(ctx, obj); } +std::vector<uint8_t> vectorFromCDR(ConstDataRange cdr) { + std::vector<uint8_t> buf(cdr.length()); + std::copy(cdr.data(), cdr.data() + cdr.length(), buf.data()); + return buf; +} template <typename T> std::vector<uint8_t> toEncryptedVector(EncryptedBinDataType dt, T t) { @@ -232,12 +248,25 @@ std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(Cons return {subType, cdrc}; } +std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedBinData(BSONElement element) { + uassert( + 6373502, "Expected binData with subtype Encrypt", element.isBinData(BinDataType::Encrypt)); + + return fromEncryptedConstDataRange(binDataToCDR(element)); +} + +template <FLETokenType TokenT> +FLEToken<TokenT> FLETokenFromCDR(ConstDataRange cdr) { + auto block = PrfBlockfromCDR(cdr); + return FLEToken<TokenT>(block); +} + /** * AEAD AES + SHA256 * Block size = 16 bytes * SHA-256 - block size = 256 bits = 32 bytes */ -// TODO (SERVER-63382) - replace with call to CTR AEAD algorithm +// TODO (SERVER-63780) - replace with call to CTR AEAD algorithm StatusWith<std::vector<uint8_t>> encryptDataWithAssociatedData(ConstDataRange key, ConstDataRange associatedData, ConstDataRange plainText) { @@ -265,11 +294,13 @@ StatusWith<std::vector<uint8_t>> encryptDataWithAssociatedData(ConstDataRange ke return {out}; } +// TODO (SERVER-63780) - replace with call to CTR algorithm, NOT CTR AEAD StatusWith<std::vector<uint8_t>> encryptData(ConstDataRange key, ConstDataRange plainText) { return encryptDataWithAssociatedData(key, ConstDataRange(0, 0), plainText); } +// TODO (SERVER-63780) - replace with call to CTR algorithm, NOT CTR AEAD StatusWith<std::vector<uint8_t>> encryptData(ConstDataRange key, uint64_t value) { std::array<char, sizeof(uint64_t)> bufValue; @@ -278,7 +309,7 @@ StatusWith<std::vector<uint8_t>> encryptData(ConstDataRange key, uint64_t value) return encryptData(key, bufValue); } -// TODO (SERVER-63382) - replace with call to CTR AEAD algorithm +// TODO (SERVER-63780) - replace with call to CTR AEAD algorithm StatusWith<std::vector<uint8_t>> decryptDataWithAssociatedData(ConstDataRange key, ConstDataRange associatedData, ConstDataRange cipherText) { @@ -309,11 +340,12 @@ StatusWith<std::vector<uint8_t>> decryptDataWithAssociatedData(ConstDataRange ke return out; } - +// TODO (SERVER-63780) - replace with call to CTR algorithm, NOT CTR AEAD StatusWith<std::vector<uint8_t>> decryptData(ConstDataRange key, ConstDataRange cipherText) { return decryptDataWithAssociatedData(key, ConstDataRange(0, 0), cipherText); } +// TODO (SERVER-63780) - replace with call to CTR algorithm, NOT CTR AEAD StatusWith<uint64_t> decryptUInt64(ConstDataRange key, ConstDataRange cipherText) { auto swPlainText = decryptData(key, cipherText); if (!swPlainText.isOK()) { @@ -705,6 +737,47 @@ BSONObj transformBSON( } /** + * Iterates through all encrypted fields. Does not return a document like doTransform. + * + * Callers can pass a function doVisit(Original bindata content, Field Name). + */ +void visitEncryptedBSON(const BSONObj& object, + const std::function<void(ConstDataRange, StringData)>& doVisit) { + std::stack<BSONObjIterator> frameStack; + + const ScopeGuard frameStackGuard([&] { + while (!frameStack.empty()) { + frameStack.pop(); + } + }); + + frameStack.emplace(BSONObjIterator(object)); + + while (frameStack.size() > 1 || frameStack.top().more()) { + uassert(6373511, + "Object too deep to be encrypted. Exceeded stack depth.", + frameStack.size() < BSONDepth::kDefaultMaxAllowableDepth); + auto& iterator = frameStack.top(); + if (iterator.more()) { + BSONElement elem = iterator.next(); + if (elem.type() == BSONType::Object) { + frameStack.emplace(BSONObjIterator(elem.Obj())); + } else if (elem.type() == BSONType::Array) { + frameStack.emplace(BSONObjIterator(elem.Obj())); + } else if (elem.isBinData(BinDataType::Encrypt)) { + int len; + const char* data(elem.binData(len)); + ConstDataRange cdr(data, len); + doVisit(cdr, elem.fieldNameStringData()); + } + } else { + frameStack.pop(); + } + } + invariant(frameStack.size() == 1); +} + +/** * Converts an encryption placeholder to FLE2InsertUpdatePayload in prepration for insert, * fndAndModify and update. */ @@ -749,6 +822,119 @@ void convertToFLE2InsertUpdateValue(FLEKeyVault* keyVault, } } +void collectEDCServerInfo(std::vector<EDCServerPayloadInfo>* pFields, + ConstDataRange cdr, + StringData fieldPath) { + + // TODO - validate acceptable types - kFLE2InsertUpdatePayload or kFLE2UnindexedEncryptedValue + // or kFLE2EqualityIndexedValue + // TODO - validate field is actually indexed in the schema? + + auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); + + auto encryptedType = encryptedTypeBinding; + uassert(6373503, + str::stream() << "Unexpected encrypted payload type: " + << static_cast<uint32_t>(encryptedType), + encryptedType == EncryptedBinDataType::kFLE2InsertUpdatePayload); + + auto iupayload = EDCClientPayload::parse(subCdr); + + uassert(6373504, + str::stream() << "Type '" << typeName(static_cast<BSONType>(iupayload.getType())) + << "' is not a valid type for FLE 2 encryption", + isValidBSONType(iupayload.getType()) && + isFLE2EqualityIndexedSupportedType(static_cast<BSONType>(iupayload.getType()))); + + pFields->push_back({std::move(iupayload), fieldPath.toString(), 0}); +} + +template <typename T> +struct ConstVectorIteratorPair { + ConstVectorIteratorPair(const std::vector<T>& vec) : it(vec.cbegin()), end(vec.cend()) {} + + typename std::vector<T>::const_iterator it; + typename std::vector<T>::const_iterator end; +}; + +struct TagInfo { + PrfBlock tag; +}; + +void convertServerPayload(std::vector<TagInfo>* pTags, + ConstVectorIteratorPair<EDCServerPayloadInfo>& it, + BSONObjBuilder* builder, + StringData fieldPath) { + + uassert(6373505, "Unexpected end of iterator", it.it != it.end); + auto payload = *(it.it); + + // TODO - validate acceptable types - kFLE2InsertUpdatePayload or kFLE2UnindexedEncryptedValue + // or kFLE2EqualityIndexedValue + // TODO - validate field is actually indexed in the schema? + + FLE2IndexedEqualityEncryptedValue sp(payload.payload, payload.count); + + uassert(6373506, + str::stream() << "Type '" << typeName(sp.bsonType) + << "' is not a valid type for FLE 2 encryption", + isFLE2EqualityIndexedSupportedType(sp.bsonType)); + + auto swEncrypted = sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( + payload.payload.getServerEncryptionToken())); + uassertStatusOK(swEncrypted); + toEncryptedBinData(fieldPath, + EncryptedBinDataType::kFLE2EqualityIndexedValue, + ConstDataRange(swEncrypted.getValue()), + builder); + + pTags->push_back({EDCServerCollection::generateTag(payload)}); + + it.it++; +} + + +BSONObj toBSON(BSONType type, ConstDataRange cdr) { + auto valueString = "value"_sd; + + // The size here is to construct a new BSON document and validate the + // total size of the object. The first four bytes is for the size of an + // int32_t, then a space for the type of the first element, then the space + // for the value string and the the 0x00 terminated field name, then the + // size of the actual data, then the last byte for the end document character, + // also 0x00. + size_t docLength = sizeof(int32_t) + 1 + valueString.size() + 1 + cdr.length() + 1; + BufBuilder builder; + builder.reserveBytes(docLength); + + uassert(ErrorCodes::BadValue, + "invalid decryption value", + docLength < std::numeric_limits<int32_t>::max()); + + builder.appendNum(static_cast<uint32_t>(docLength)); + builder.appendChar(static_cast<uint8_t>(type)); + builder.appendStr(valueString, true); + builder.appendBuf(cdr.data(), cdr.length()); + builder.appendChar('\0'); + + ConstDataRangeCursor cdc = ConstDataRangeCursor(ConstDataRange(builder.buf(), builder.len())); + BSONObj elemWrapped = cdc.readAndAdvance<Validated<BSONObj>>(); + return elemWrapped.getOwned(); +} + + +void decryptField(FLEKeyVault* keyVault, + ConstDataRange cdr, + BSONObjBuilder* builder, + StringData fieldPath) { + + auto pair = FLEClientCrypto::decrypt(cdr, keyVault); + + BSONObj obj = toBSON(pair.first, pair.second); + + builder->appendAs(obj.firstElement(), fieldPath); +} + } // namespace @@ -886,6 +1072,63 @@ BSONObj FLEClientCrypto::generateInsertOrUpdateFromPlaceholders(const BSONObj& o return ret; } + +std::pair<BSONType, std::vector<uint8_t>> FLEClientCrypto::decrypt(BSONElement element, + FLEKeyVault* keyVault) { + auto pair = fromEncryptedBinData(element); + + return FLEClientCrypto::decrypt(pair.second, keyVault); +} + +std::pair<BSONType, std::vector<uint8_t>> FLEClientCrypto::decrypt(ConstDataRange cdr, + FLEKeyVault* keyVault) { + auto pair = fromEncryptedConstDataRange(cdr); + + if (pair.first == EncryptedBinDataType::kFLE2EqualityIndexedValue) { + auto indexKeyId = + uassertStatusOK(FLE2IndexedEqualityEncryptedValue::readKeyId(pair.second)); + + auto indexKey = keyVault->getIndexKeyById(indexKeyId); + + auto serverDataToken = + FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); + + auto ieev = uassertStatusOK( + FLE2IndexedEqualityEncryptedValue::decryptAndParse(serverDataToken, pair.second)); + + auto userCipherText = ieev.clientEncryptedValue; + + auto userKeyId = uassertStatusOK(KeyIdAndValue::readKeyId(userCipherText)); + + auto userKey = keyVault->getUserKeyById(userKeyId); + + auto userData = uassertStatusOK(KeyIdAndValue::decrypt(userKey.key, userCipherText)); + + return {ieev.bsonType, userData}; + + } else { + uasserted(6373507, "Not supported"); + } + + return {EOO, std::vector<uint8_t>()}; +} + +BSONObj FLEClientCrypto::decryptDocument(BSONObj& doc, FLEKeyVault* keyVault) { + + BSONObjBuilder builder; + + // TODO - validate acceptable types - kFLE2UnindexedEncryptedValue or kFLE2EqualityIndexedValue + // kFLE2InsertUpdatePayload? + auto obj = transformBSON( + doc, [keyVault](ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { + decryptField(keyVault, cdr, builder, fieldPath); + }); + + builder.appendElements(obj); + + return builder.obj(); +} + PrfBlock ESCCollection::generateId(ESCTwiceDerivedTagToken tagToken, boost::optional<uint64_t> index) { if (index.has_value()) { @@ -1131,7 +1374,6 @@ BSONObj ECOCollection::generateDocument(StringData fieldName, ConstDataRange pay BSONObjBuilder builder; builder.append(kFieldName, fieldName); toBinData(kValue, payload, &builder); - return builder.obj(); } @@ -1150,4 +1392,250 @@ ECOCCompactionDocument ECOCollection::parseAndDecrypt(BSONObj& doc, ECOCToken to return ret; } +FLE2IndexedEqualityEncryptedValue::FLE2IndexedEqualityEncryptedValue( + FLE2InsertUpdatePayload payload, uint64_t counter) + : edc(FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( + payload.getEdcDerivedToken())), + esc(FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( + payload.getEscDerivedToken())), + ecc(FLETokenFromCDR<FLETokenType::ECCDerivedFromDataTokenAndContentionFactorToken>( + payload.getEccDerivedToken())), + count(counter), + bsonType(static_cast<BSONType>(payload.getType())), + indexKeyId(payload.getIndexKeyId()), + clientEncryptedValue(vectorFromCDR(payload.getValue())) { + uassert(6373508, + "Invalid BSON Type in FLE2InsertUpdatePayload", + isValidBSONType(payload.getType())); +} + +FLE2IndexedEqualityEncryptedValue::FLE2IndexedEqualityEncryptedValue( + EDCDerivedFromDataTokenAndContentionFactorToken edcParam, + ESCDerivedFromDataTokenAndContentionFactorToken escParam, + ECCDerivedFromDataTokenAndContentionFactorToken eccParam, + uint64_t countParam, + BSONType typeParam, + UUID indexKeyIdParam, + std::vector<uint8_t> clientEncryptedValueParam) + : edc(edcParam), + esc(escParam), + ecc(eccParam), + count(countParam), + bsonType(typeParam), + indexKeyId(indexKeyIdParam), + clientEncryptedValue(clientEncryptedValueParam) {} + +StatusWith<UUID> FLE2IndexedEqualityEncryptedValue::readKeyId( + ConstDataRange serializedServerValue) { + ConstDataRangeCursor baseCdrc(serializedServerValue); + + auto swKeyId = baseCdrc.readAndAdvanceNoThrow<UUIDBuf>(); + if (!swKeyId.isOK()) { + return {swKeyId.getStatus()}; + } + + return UUID::fromCDR(swKeyId.getValue()); +} + +StatusWith<FLE2IndexedEqualityEncryptedValue> FLE2IndexedEqualityEncryptedValue::decryptAndParse( + ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { + + ConstDataRangeCursor serializedServerCdrc(serializedServerValue); + + auto swIndexKeyId = serializedServerCdrc.readAndAdvanceNoThrow<UUIDBuf>(); + if (!swIndexKeyId.isOK()) { + return {swIndexKeyId.getStatus()}; + } + + UUID indexKey = UUID::fromCDR(swIndexKeyId.getValue()); + + auto swBsonType = serializedServerCdrc.readAndAdvanceNoThrow<uint8_t>(); + if (!swBsonType.isOK()) { + return {swBsonType.getStatus()}; + } + + uassert(6373509, + "Invalid BSON Type in FLE2InsertUpdatePayload", + isValidBSONType(swBsonType.getValue())); + + auto type = static_cast<BSONType>(swBsonType.getValue()); + + auto swVec = decryptData(token.toCDR(), serializedServerCdrc); + if (!swVec.isOK()) { + return swVec.getStatus(); + } + + auto data = swVec.getValue(); + + ConstDataRangeCursor serverEncryptedValueCdrc(data); + + auto swLength = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); + if (!swLength.isOK()) { + return {swLength.getStatus()}; + } + + std::uint64_t length = swLength.getValue(); + + auto start = serverEncryptedValueCdrc.data(); + + auto advance = serverEncryptedValueCdrc.advanceNoThrow(length); + if (!advance.isOK()) { + return advance; + } + + std::vector<uint8_t> cipherText(length); + + std::copy(start, start + length, cipherText.data()); + + auto swCount = serverEncryptedValueCdrc.readAndAdvanceNoThrow<LittleEndian<std::uint64_t>>(); + if (!swCount.isOK()) { + return {swCount.getStatus()}; + } + + auto swEdc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); + if (!swEdc.isOK()) { + return swEdc.getStatus(); + } + + auto swEsc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); + if (!swEsc.isOK()) { + return swEsc.getStatus(); + } + + auto swEcc = serverEncryptedValueCdrc.readAndAdvanceNoThrow<PrfBlock>(); + if (!swEcc.isOK()) { + return swEcc.getStatus(); + } + + return FLE2IndexedEqualityEncryptedValue( + EDCDerivedFromDataTokenAndContentionFactorToken(swEdc.getValue()), + ESCDerivedFromDataTokenAndContentionFactorToken(swEsc.getValue()), + ECCDerivedFromDataTokenAndContentionFactorToken(swEcc.getValue()), + swCount.getValue(), + type, + indexKey, + std::move(cipherText)); +} + + +StatusWith<std::vector<uint8_t>> FLE2IndexedEqualityEncryptedValue::serialize( + ServerDataEncryptionLevel1Token token) { + BufBuilder builder(clientEncryptedValue.size() + sizeof(uint64_t) * 2 + sizeof(PrfBlock) * 3); + + + builder.appendNum(LittleEndian<uint64_t>(clientEncryptedValue.size())); + + builder.appendBuf(clientEncryptedValue.data(), clientEncryptedValue.size()); + + builder.appendNum(LittleEndian<uint64_t>(count)); + + builder.appendStruct(edc.data); + + builder.appendStruct(esc.data); + + builder.appendStruct(ecc.data); + + dassert(builder.len() == + static_cast<int>(clientEncryptedValue.size() + sizeof(uint64_t) * 2 + + sizeof(PrfBlock) * 3)); + + auto swEncryptedData = encryptData(token.toCDR(), ConstDataRange(builder.buf(), builder.len())); + if (!swEncryptedData.isOK()) { + return swEncryptedData; + } + + auto cdrKeyId = indexKeyId.toCDR(); + auto serverEncryptedValue = swEncryptedData.getValue(); + + std::vector<uint8_t> serializedServerValue(serverEncryptedValue.size() + cdrKeyId.length() + 1); + + std::copy(cdrKeyId.data(), cdrKeyId.data() + cdrKeyId.length(), serializedServerValue.begin()); + uint8_t bsonTypeByte = bsonType; + std::copy( + &bsonTypeByte, (&bsonTypeByte) + 1, serializedServerValue.begin() + cdrKeyId.length()); + std::copy(serverEncryptedValue.begin(), + serverEncryptedValue.end(), + serializedServerValue.begin() + cdrKeyId.length() + 1); + + return serializedServerValue; +} + + +std::vector<EDCServerPayloadInfo> EDCServerCollection::getEncryptedFieldInfo(BSONObj& obj) { + std::vector<EDCServerPayloadInfo> fields; + // TODO (SERVER-63736) - Validate only fields listed in EncryptedFieldConfig are indexed + + visitEncryptedBSON(obj, [&fields](ConstDataRange cdr, StringData fieldPath) { + collectEDCServerInfo(&fields, cdr, fieldPath); + }); + + return fields; +} + +PrfBlock EDCServerCollection::generateTag(EDCTwiceDerivedToken edcTwiceDerived, FLECounter count) { + return prf(edcTwiceDerived.toCDR(), count); +} + +PrfBlock EDCServerCollection::generateTag(const EDCServerPayloadInfo& payload) { + auto token = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( + payload.payload.getEdcDerivedToken()); + auto edcTwiceDerived = FLETwiceDerivedTokenGenerator::generateEDCTwiceDerivedToken(token); + return generateTag(edcTwiceDerived, payload.count); +} + +BSONObj EDCServerCollection::finalizeForInsert( + BSONObj& doc, const std::vector<EDCServerPayloadInfo>& serverPayload) { + + std::vector<TagInfo> tags; + // TODO - improve size estimate after range is supported since it no longer be 1 to 1 + tags.reserve(serverPayload.size()); + + ConstVectorIteratorPair<EDCServerPayloadInfo> it(serverPayload); + + // First: transform all the markings + auto obj = transformBSON( + doc, [&tags, &it](ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldPath) { + convertServerPayload(&tags, it, builder, fieldPath); + }); + + BSONObjBuilder builder; + + // Second: reuse an existing array if present + bool appendElements = true; + for (const auto& element : obj) { + if (element.fieldNameStringData() == kSafeContent) { + uassert(6373510, + str::stream() << "Field '" << kSafeContent << "' was found but not an array", + element.type() == Array); + BSONArrayBuilder subBuilder(builder.subarrayStart(kSafeContent)); + + // Append existing array elements + for (const auto& arrayElement : element.Obj()) { + subBuilder.append(arrayElement); + } + + // Add new tags + for (auto const& tag : tags) { + appendTag(tag.tag, &subBuilder); + } + + appendElements = false; + } else { + builder.append(element); + } + } + + // Third: append the tags array if it does not exist + if (appendElements) { + BSONArrayBuilder subBuilder(builder.subarrayStart(kSafeContent)); + + for (auto const& tag : tags) { + appendTag(tag.tag, &subBuilder); + } + } + + return builder.obj(); +} + + } // namespace mongo diff --git a/src/mongo/crypto/fle_crypto.h b/src/mongo/crypto/fle_crypto.h index 56d1218a25c..f5fef90cf88 100644 --- a/src/mongo/crypto/fle_crypto.h +++ b/src/mongo/crypto/fle_crypto.h @@ -42,6 +42,8 @@ #include "mongo/bson/bsonelement.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsontypes.h" +#include "mongo/crypto/encryption_fields_gen.h" +#include "mongo/crypto/fle_field_schema_gen.h" #include "mongo/util/uuid.h" @@ -688,6 +690,18 @@ public: FLEUserKeyAndId userKey, FLECounter counter); + + /** + * Explicit decrypt a single value into type and value + * + * Supports decrypting FLE2IndexedEqualityEncryptedValue + */ + static std::pair<BSONType, std::vector<uint8_t>> decrypt(ConstDataRange cdr, + FLEKeyVault* keyVault); + + static std::pair<BSONType, std::vector<uint8_t>> decrypt(BSONElement element, + FLEKeyVault* keyVault); + /** * Generates a client-side payload that is sent to the server. * @@ -706,6 +720,11 @@ public: */ static BSONObj generateInsertOrUpdateFromPlaceholders(const BSONObj& obj, FLEKeyVault* keyVault); + + /** + * Decrypts a document. Only supports FLE2. + */ + static BSONObj decryptDocument(BSONObj& doc, FLEKeyVault* keyVault); }; /* @@ -715,8 +734,8 @@ public: * ECCDerivedFromDataTokenAndContentionFactorToken) * * struct { - * uint8_t[64] esc; - * uint8_t[64] ecc; + * uint8_t[32] esc; + * uint8_t[32] ecc; * } */ struct EncryptedStateCollectionTokens { @@ -766,4 +785,98 @@ public: }; +/** + * Class to read/write FLE2 Equality Indexed Encrypted Values + * + * Fields are encrypted with the following: + * + * struct { + * uint8_t fle_blob_subtype = 7; + * uint8_t key_uuid[16]; + * uint8 original_bson_type; + * ciphertext[ciphertext_length]; + * } + * + * Encrypt(ServerDataEncryptionLevel1Token, Struct(K_KeyId, v, count, d, s, c)) + * + * struct { + * uint8_t[length] cipherText; // UserKeyId + Encrypt(K_KeyId, value), + * uint64_t counter; + * uint8_t[32] edc; // EDCDerivedFromDataTokenAndContentionFactorToken + * uint8_t[32] esc; // ESCDerivedFromDataTokenAndContentionFactorToken + * uint8_t[32] ecc; // ECCDerivedFromDataTokenAndContentionFactorToken + *} + */ +struct FLE2IndexedEqualityEncryptedValue { + FLE2IndexedEqualityEncryptedValue(FLE2InsertUpdatePayload payload, uint64_t counter); + + FLE2IndexedEqualityEncryptedValue(EDCDerivedFromDataTokenAndContentionFactorToken edcParam, + ESCDerivedFromDataTokenAndContentionFactorToken escParam, + ECCDerivedFromDataTokenAndContentionFactorToken eccParam, + uint64_t countParam, + BSONType typeParam, + UUID indexKeyIdParam, + std::vector<uint8_t> serializedServerValueParam); + + static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( + ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); + + /** + * Read the key id from the payload. + */ + static StatusWith<UUID> readKeyId(ConstDataRange serializedServerValue); + + StatusWith<std::vector<uint8_t>> serialize(ServerDataEncryptionLevel1Token token); + + EDCDerivedFromDataTokenAndContentionFactorToken edc; + ESCDerivedFromDataTokenAndContentionFactorToken esc; + ECCDerivedFromDataTokenAndContentionFactorToken ecc; + uint64_t count; + BSONType bsonType; + UUID indexKeyId; + std::vector<uint8_t> clientEncryptedValue; +}; + + +struct EDCServerPayloadInfo { + FLE2InsertUpdatePayload payload; + + std::string fieldPathName; + uint64_t count; +}; + +/** + * Manipulates the EDC collection. + * + * To finalize a document for insertion + * + * 1. Get all the encrypted fields that need counters via getEncryptedFieldInfo() + * 2. Choose counters + * 3. Finalize the insertion with finalizeForInsert(). + */ +class EDCServerCollection { +public: + /** + * Get information about all FLE2InsertUpdatePayload payloads + */ + static std::vector<EDCServerPayloadInfo> getEncryptedFieldInfo(BSONObj& obj); + + /** + * Generate a search tag + * + * HMAC(EDCTwiceDerivedToken, count) + */ + static PrfBlock generateTag(EDCTwiceDerivedToken edcTwiceDerived, FLECounter count); + static PrfBlock generateTag(const EDCServerPayloadInfo& payload); + + /** + * Consumes a payload from a MongoDB client for insert. + * + * Converts FLE2InsertUpdatePayload to a final insert payload and updates __safeContent__ with + * new tags. + */ + static BSONObj finalizeForInsert(BSONObj& doc, + const std::vector<EDCServerPayloadInfo>& serverPayload); +}; + } // namespace mongo diff --git a/src/mongo/crypto/fle_crypto_test.cpp b/src/mongo/crypto/fle_crypto_test.cpp index 7c35914fc48..e9da2a86c44 100644 --- a/src/mongo/crypto/fle_crypto_test.cpp +++ b/src/mongo/crypto/fle_crypto_test.cpp @@ -34,11 +34,13 @@ #include <algorithm> #include <iostream> #include <limits> +#include <stack> #include <string> #include <tuple> #include <vector> #include "mongo/base/data_range.h" +#include "mongo/base/data_type_validated.h" #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" @@ -121,6 +123,22 @@ static UUID userKeyId = uassertStatusOK(UUID::parse(kUserKeyId.toString())); std::vector<char> testValue = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19}; std::vector<char> testValue2 = {0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29}; +class TestKeyVault : public FLEKeyVault { +public: + KeyMaterial getKey(UUID uuid) override; +}; + +KeyMaterial TestKeyVault::getKey(UUID uuid) { + if (uuid == indexKeyId) { + return indexKey.data; + } else if (uuid == userKeyId) { + return userKey.data; + } else { + FAIL("not implemented"); + return KeyMaterial{}; + } +} + TEST(FLETokens, TestVectors) { @@ -466,5 +484,410 @@ TEST(FLE_ESC, EmuBinary2) { ASSERT_EQ(i, 5); } +std::vector<char> generatePlaceholder(BSONElement value) { + FLE2EncryptionPlaceholder ep; + + ep.setAlgorithm(mongo::Fle2AlgorithmInt::kEquality); + ep.setUserKeyId(userKeyId); + ep.setIndexKeyId(indexKeyId); + ep.setValue(value); + ep.setType(mongo::Fle2PlaceholderType::kInsert); + ep.setMaxContentionCounter(0); + + BSONObj obj = ep.toBSON(); + + std::vector<char> v; + v.resize(obj.objsize() + 1); + v[0] = static_cast<uint8_t>(EncryptedBinDataType::kFLE2Placeholder); + std::copy(obj.objdata(), obj.objdata() + obj.objsize(), v.begin() + 1); + return v; +} + +BSONObj encryptDocument(BSONObj obj, FLEKeyVault* keyVault) { + auto result = FLEClientCrypto::generateInsertOrUpdateFromPlaceholders(obj, keyVault); + + // Start Server Side + auto serverPayload = EDCServerCollection::getEncryptedFieldInfo(result); + + ASSERT_EQ(serverPayload.size(), 1); + + // TODO set count based on EmuBinary + for (auto& payload : serverPayload) { + payload.count = 1; + } + + // Finalize document for insert + auto finalDoc = EDCServerCollection::finalizeForInsert(result, serverPayload); + ASSERT_EQ(finalDoc["__safeContent__"].type(), Array); + return finalDoc; +} + +void roundTripTest(BSONObj doc, BSONType type) { + + auto element = doc.firstElement(); + ASSERT_EQ(element.type(), type); + + + TestKeyVault keyVault; + + auto inputDoc = BSON("plainText" + << "sample" + << "encrypted" << element); + + auto buf = generatePlaceholder(element); + BSONObjBuilder builder; + builder.append("plainText", "sample"); + builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); + + auto finalDoc = encryptDocument(builder.obj(), &keyVault); + + ASSERT_EQ(finalDoc["plainText"].type(), String); + ASSERT_EQ(finalDoc["encrypted"].type(), BinData); + ASSERT_TRUE(finalDoc["encrypted"].isBinData(BinDataType::Encrypt)); + + // Decrypt document + auto decryptedDoc = FLEClientCrypto::decryptDocument(finalDoc, &keyVault); + + // Remove this so the round-trip is clean + decryptedDoc = decryptedDoc.removeField("__safeContent__"); + + ASSERT_BSONOBJ_EQ(inputDoc, decryptedDoc); +} + +TEST(FLE_EDC, Allowed_Types) { + roundTripTest(BSON("sample" + << "value123"), + String); + roundTripTest(BSON("sample" << BSONBinData( + testValue.data(), testValue.size(), BinDataType::BinDataGeneral)), + BinData); + roundTripTest(BSON("sample" << OID()), jstOID); + + + roundTripTest(BSON("sample" << false), Bool); + roundTripTest(BSON("sample" << true), Bool); + roundTripTest(BSON("sample" << Date_t()), Date); + roundTripTest(BSON("sample" << BSONRegEx("value1", "value2")), RegEx); + roundTripTest(BSON("sample" << 123456), NumberInt); + roundTripTest(BSON("sample" << Timestamp()), bsonTimestamp); + roundTripTest(BSON("sample" << 12345678901234567LL), NumberLong); + roundTripTest(BSON("sample" << BSONCode("value")), Code); +} + +void illegalBSONType(BSONObj doc, BSONType type) { + auto element = doc.firstElement(); + if (isValidBSONType(type)) { + ASSERT_EQ(element.type(), type); + } + + TestKeyVault keyVault; + + auto buf = generatePlaceholder(element); + BSONObjBuilder builder; + builder.append("plainText", "sample"); + builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); + BSONObj obj = builder.obj(); + + ASSERT_THROWS_CODE(FLEClientCrypto::generateInsertOrUpdateFromPlaceholders(obj, &keyVault), + DBException, + 6338602); +} + +TEST(FLE_EDC, Disallowed_Types) { + illegalBSONType(BSON("sample" << 123.456), NumberDouble); + illegalBSONType(BSON("sample" << Decimal128()), NumberDecimal); + + illegalBSONType(BSON("sample" << MINKEY), MinKey); + + illegalBSONType(BSON("sample" << BSON("nested" + << "value")), + Object); + illegalBSONType(BSON("sample" << BSON_ARRAY(1 << 23)), Array); + + illegalBSONType(BSON("sample" << BSONUndefined), Undefined); + illegalBSONType(BSON("sample" << BSONNULL), jstNULL); + illegalBSONType(BSON("sample" << BSONDBRef("value1", OID())), DBRef); + illegalBSONType(BSON("sample" << BSONSymbol("value")), Symbol); + illegalBSONType(BSON("sample" << BSONCodeWScope("value", + BSON("code" + << "something"))), + CodeWScope); + + illegalBSONType(BSON("sample" << MAXKEY), MaxKey); + + + uint8_t fakeBSONType = 42; + ASSERT_FALSE(isValidBSONType(fakeBSONType)); + illegalBSONType(BSON("sample" << 123.456), static_cast<BSONType>(fakeBSONType)); +} + + +template <typename T> +T parseFromCDR(ConstDataRange cdr) { + ConstDataRangeCursor cdc(cdr); + auto swObj = cdc.readAndAdvanceNoThrow<Validated<BSONObj>>(); + + uassertStatusOK(swObj); + + BSONObj obj = swObj.getValue(); + + IDLParserErrorContext ctx("root"); + return T::parse(ctx, obj); +} + +template <typename T> +std::vector<uint8_t> toEncryptedVector(EncryptedBinDataType dt, T t) { + BSONObj obj = t.toBSON(); + + std::vector<uint8_t> buf(obj.objsize() + 1); + buf[0] = static_cast<uint8_t>(dt); + + std::copy(obj.objdata(), obj.objdata() + obj.objsize(), buf.data() + 1); + + return buf; +} + +template <typename T> +void toEncryptedBinData(StringData field, EncryptedBinDataType dt, T t, BSONObjBuilder* builder) { + auto buf = toEncryptedVector(dt, t); + + builder->appendBinData(field, buf.size(), BinDataType::Encrypt, buf.data()); +} + + +std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(ConstDataRange cdr) { + ConstDataRangeCursor cdrc(cdr); + + uint8_t subTypeByte = cdrc.readAndAdvance<uint8_t>(); + + auto subType = EncryptedBinDataType_parse(IDLParserErrorContext("subtype"), subTypeByte); + return {subType, cdrc}; +} + + +BSONObj transformBSON( + const BSONObj& object, + const std::function<void(ConstDataRange, BSONObjBuilder*, StringData)>& doTransform) { + struct IteratorState { + BSONObjIterator iter; + BSONObjBuilder builder; + }; + + std::stack<IteratorState> frameStack; + + const ScopeGuard frameStackGuard([&] { + while (!frameStack.empty()) { + frameStack.pop(); + } + }); + + frameStack.push({BSONObjIterator(object), BSONObjBuilder()}); + + while (frameStack.size() > 1 || frameStack.top().iter.more()) { + ASSERT(frameStack.size() < BSONDepth::kDefaultMaxAllowableDepth); + auto& [iterator, builder] = frameStack.top(); + if (iterator.more()) { + BSONElement elem = iterator.next(); + if (elem.type() == BSONType::Object) { + frameStack.push({BSONObjIterator(elem.Obj()), + BSONObjBuilder(builder.subobjStart(elem.fieldNameStringData()))}); + } else if (elem.type() == BSONType::Array) { + frameStack.push( + {BSONObjIterator(elem.Obj()), + BSONObjBuilder(builder.subarrayStart(elem.fieldNameStringData()))}); + } else if (elem.isBinData(BinDataType::Encrypt)) { + int len; + const char* data(elem.binData(len)); + ConstDataRange cdr(data, len); + doTransform(cdr, &builder, elem.fieldNameStringData()); + } else { + builder.append(elem); + } + } else { + frameStack.pop(); + } + } + invariant(frameStack.size() == 1); + return frameStack.top().builder.obj(); +} + +void disallowedEqualityPayloadType(BSONType type) { + auto doc = BSON("sample" << 123456); + auto element = doc.firstElement(); + + TestKeyVault keyVault; + + + auto inputDoc = BSON("plainText" + << "sample" + << "encrypted" << element); + + auto buf = generatePlaceholder(element); + BSONObjBuilder builder; + builder.append("plainText", "sample"); + builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); + BSONObj obj = builder.obj(); + + auto result = FLEClientCrypto::generateInsertOrUpdateFromPlaceholders(obj, &keyVault); + + // Since FLEClientCrypto::generateInsertOrUpdateFromPlaceholders validates the type is correct, + // we send an allowed type and then change the type to something that is not allowed + result = transformBSON( + result, + [type](ConstDataRange cdr, BSONObjBuilder* builder, StringData fieldNameToSerialize) { + auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); + + + auto iup = parseFromCDR<FLE2InsertUpdatePayload>(subCdr); + + iup.setType(type); + toEncryptedBinData( + fieldNameToSerialize, EncryptedBinDataType::kFLE2InsertUpdatePayload, iup, builder); + }); + + + // Start Server Side + ASSERT_THROWS_CODE(EDCServerCollection::getEncryptedFieldInfo(result), DBException, 6373504); +} + +TEST(FLE_EDC, Disallowed_Types_FLE2InsertUpdatePayload) { + disallowedEqualityPayloadType(NumberDouble); + disallowedEqualityPayloadType(NumberDecimal); + + disallowedEqualityPayloadType(MinKey); + + disallowedEqualityPayloadType(Object); + disallowedEqualityPayloadType(Array); + + disallowedEqualityPayloadType(Undefined); + disallowedEqualityPayloadType(jstNULL); + disallowedEqualityPayloadType(DBRef); + disallowedEqualityPayloadType(Symbol); + disallowedEqualityPayloadType(CodeWScope); + + disallowedEqualityPayloadType(MaxKey); + + uint8_t fakeBSONType = 42; + ASSERT_FALSE(isValidBSONType(fakeBSONType)); + disallowedEqualityPayloadType(static_cast<BSONType>(fakeBSONType)); +} + + +TEST(FLE_EDC, ServerSide_Payloads) { + auto doc = BSON("sample" << 123456); + auto element = doc.firstElement(); + + auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); + + auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey); + auto serverEncryptToken = + FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey); + auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); + auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); + auto eccToken = FLECollectionTokenGenerator::generateECCToken(collectionToken); + auto ecocToken = FLECollectionTokenGenerator::generateECOCToken(collectionToken); + + FLECounter counter = 0; + + + EDCDerivedFromDataToken edcDatakey = + FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); + ESCDerivedFromDataToken escDatakey = + FLEDerivedFromDataTokenGenerator::generateESCDerivedFromDataToken(escToken, value); + ECCDerivedFromDataToken eccDatakey = + FLEDerivedFromDataTokenGenerator::generateECCDerivedFromDataToken(eccToken, value); + + + ESCDerivedFromDataTokenAndContentionFactorToken escDataCounterkey = + FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: + generateESCDerivedFromDataTokenAndContentionFactorToken(escDatakey, counter); + ECCDerivedFromDataTokenAndContentionFactorToken eccDataCounterkey = + FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: + generateECCDerivedFromDataTokenAndContentionFactorToken(eccDatakey, counter); + + FLE2InsertUpdatePayload iupayload; + + + iupayload.setEdcDerivedToken(edcDatakey.toCDR()); + iupayload.setEscDerivedToken(escDatakey.toCDR()); + iupayload.setEccDerivedToken(eccDatakey.toCDR()); + iupayload.setServerEncryptionToken(serverEncryptToken.toCDR()); + + auto swEncryptedTokens = + EncryptedStateCollectionTokens(escDataCounterkey, eccDataCounterkey).serialize(ecocToken); + uassertStatusOK(swEncryptedTokens); + iupayload.setEncryptedTokens(swEncryptedTokens.getValue()); + + std::vector<uint8_t> cipherText = {0x1, 0x2, 0x3, 0x4, 0x5}; + iupayload.setValue(cipherText); + iupayload.setType(element.type()); + + FLE2IndexedEqualityEncryptedValue serverPayload(iupayload, 123456); + + auto swBuf = serverPayload.serialize(serverEncryptToken); + ASSERT_OK(swBuf.getStatus()); + + auto swServerPayload = + FLE2IndexedEqualityEncryptedValue::decryptAndParse(serverEncryptToken, swBuf.getValue()); + + ASSERT_OK(swServerPayload.getStatus()); + auto sp = swServerPayload.getValue(); + ASSERT_EQ(sp.edc, serverPayload.edc); + ASSERT_EQ(sp.esc, serverPayload.esc); + ASSERT_EQ(sp.ecc, serverPayload.ecc); + ASSERT_EQ(sp.count, serverPayload.count); + ASSERT(sp.clientEncryptedValue == serverPayload.clientEncryptedValue); + ASSERT(cipherText == serverPayload.clientEncryptedValue); +} + +TEST(FLE_EDC, DuplicateSafeContent_CompatibleType) { + + TestKeyVault keyVault; + + auto doc = BSON("value" + << "123456"); + auto element = doc.firstElement(); + auto inputDoc = BSON("__safeContent__" << BSON_ARRAY(1 << 2 << 4) << "encrypted" << element); + + auto buf = generatePlaceholder(element); + BSONObjBuilder builder; + builder.append("__safeContent__", BSON_ARRAY(1 << 2 << 4)); + builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); + + auto finalDoc = encryptDocument(builder.obj(), &keyVault); + + ASSERT_EQ(finalDoc["__safeContent__"].type(), Array); + ASSERT_EQ(finalDoc["encrypted"].type(), BinData); + ASSERT_TRUE(finalDoc["encrypted"].isBinData(BinDataType::Encrypt)); + + // Decrypt document + auto decryptedDoc = FLEClientCrypto::decryptDocument(finalDoc, &keyVault); + + std::cout << "Final Doc: " << decryptedDoc << std::endl; + + auto elements = finalDoc["__safeContent__"].Array(); + ASSERT_EQ(elements.size(), 4); + ASSERT_EQ(elements[0].safeNumberInt(), 1); + ASSERT_EQ(elements[1].safeNumberInt(), 2); + ASSERT_EQ(elements[2].safeNumberInt(), 4); + ASSERT(elements[3].type() == BinData); +} + + +TEST(FLE_EDC, DuplicateSafeContent_IncompatibleType) { + + TestKeyVault keyVault; + + auto doc = BSON("value" + << "123456"); + auto element = doc.firstElement(); + + auto buf = generatePlaceholder(element); + BSONObjBuilder builder; + builder.append("__safeContent__", 123456); + builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); + + ASSERT_THROWS_CODE(encryptDocument(builder.obj(), &keyVault), DBException, 6373510); +} } // namespace mongo diff --git a/src/mongo/rpc/rewrite_state_change_errors.cpp b/src/mongo/rpc/rewrite_state_change_errors.cpp index f3928a7f4bb..d5e31ce1982 100644 --- a/src/mongo/rpc/rewrite_state_change_errors.cpp +++ b/src/mongo/rpc/rewrite_state_change_errors.cpp @@ -201,7 +201,7 @@ boost::optional<BSONObj> rewriteDocument(const BSONObj& doc, OperationContext* o } if (mutableDoc) { - LOGV2_DEBUG(1, 5054900, "Rewrote state change error", "code"_attr = oldCode); + LOGV2_DEBUG(5054900, 1, "Rewrote state change error", "code"_attr = oldCode); return mutableDoc->getObject(); } return {}; |