diff options
author | Sergey Galtsev <sergey.galtsev@mongodb.com> | 2022-08-30 14:53:21 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-09-07 21:29:14 +0000 |
commit | d8b0d143d2e4a03cc47bb7bb78f97fc8201f17ed (patch) | |
tree | f100304ba5aee05b991d269dfba0fa65f2e20ca2 | |
parent | b2745b93a9548266b61de11c67e97fffc791b55a (diff) | |
download | mongo-d8b0d143d2e4a03cc47bb7bb78f97fc8201f17ed.tar.gz |
SERVER-67755 fle2 range - implement insert
-rw-r--r-- | buildscripts/resmokeconfig/suites/fle2.yml | 2 | ||||
-rw-r--r-- | buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml | 2 | ||||
-rw-r--r-- | buildscripts/resmokeconfig/suites/fle2_sharding.yml | 2 | ||||
-rw-r--r-- | buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml | 2 | ||||
-rw-r--r-- | jstests/fle2/libs/encrypted_client_util.js | 27 | ||||
-rw-r--r-- | src/mongo/crypto/encryption_fields.idl | 2 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto.cpp | 139 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto.h | 3 | ||||
-rw-r--r-- | src/mongo/crypto/fle_crypto_test.cpp | 56 | ||||
-rw-r--r-- | src/mongo/crypto/fle_field_schema.idl | 2 | ||||
-rw-r--r-- | src/mongo/crypto/fle_fields_util.cpp | 10 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_options_validation.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/fle_crud.cpp | 114 | ||||
-rw-r--r-- | src/mongo/db/matcher/expression_type.h | 1 | ||||
-rw-r--r-- | src/mongo/db/matcher/expression_type_test.cpp | 1 | ||||
-rw-r--r-- | src/mongo/shell/shell_options.cpp | 1 |
16 files changed, 243 insertions, 125 deletions
diff --git a/buildscripts/resmokeconfig/suites/fle2.yml b/buildscripts/resmokeconfig/suites/fle2.yml index 6d734a18857..1d123c5fcd4 100644 --- a/buildscripts/resmokeconfig/suites/fle2.yml +++ b/buildscripts/resmokeconfig/suites/fle2.yml @@ -12,6 +12,7 @@ executor: config: shell_options: eval: "testingReplication = true; testingFLESharding = false;" + setShellParameter: featureFlagFLE2Range=true hooks: # We don't execute dbHash or oplog consistency checks since there is only a single replica set # node. @@ -23,5 +24,6 @@ executor: mongod_options: set_parameters: enableTestCommands: 1 + featureFlagFLE2Range: true # Use a 2-node replica set. num_nodes: 2 diff --git a/buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml b/buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml index a06189c6a39..3b39160afe4 100644 --- a/buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml +++ b/buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml @@ -16,6 +16,7 @@ executor: config: shell_options: eval: "testingReplication = true; testingFLESharding = false;" + setShellParameter: featureFlagFLE2Range=true hooks: # We don't execute dbHash or oplog consistency checks since there is only a single replica set # node. @@ -27,6 +28,7 @@ executor: mongod_options: set_parameters: enableTestCommands: 1 + featureFlagFLE2Range: true internalQueryFLEAlwaysUseEncryptedCollScanMode: 1 # Use a 2-node replica set. num_nodes: 2 diff --git a/buildscripts/resmokeconfig/suites/fle2_sharding.yml b/buildscripts/resmokeconfig/suites/fle2_sharding.yml index fdcea34d824..a7768069563 100644 --- a/buildscripts/resmokeconfig/suites/fle2_sharding.yml +++ b/buildscripts/resmokeconfig/suites/fle2_sharding.yml @@ -13,6 +13,7 @@ executor: config: shell_options: eval: "testingReplication = false; testingFLESharding = true;" + setShellParameter: featureFlagFLE2Range=true hooks: - class: CheckReplDBHash - class: ValidateCollections @@ -26,6 +27,7 @@ executor: mongod_options: set_parameters: enableTestCommands: 1 + featureFlagFLE2Range: true num_rs_nodes_per_shard: 2 enable_sharding: - test diff --git a/buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml b/buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml index 92be832a5ef..32f14ce8dcc 100644 --- a/buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml +++ b/buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml @@ -17,6 +17,7 @@ executor: config: shell_options: eval: "testingReplication = false; testingFLESharding = true;" + setShellParameter: featureFlagFLE2Range=true hooks: - class: CheckReplDBHash - class: ValidateCollections @@ -32,6 +33,7 @@ executor: set_parameters: enableTestCommands: 1 internalQueryFLEAlwaysUseEncryptedCollScanMode: 1 + featureFlagFLE2Range: true num_rs_nodes_per_shard: 2 enable_sharding: - test diff --git a/jstests/fle2/libs/encrypted_client_util.js b/jstests/fle2/libs/encrypted_client_util.js index b10c2f606a3..91a08f8e732 100644 --- a/jstests/fle2/libs/encrypted_client_util.js +++ b/jstests/fle2/libs/encrypted_client_util.js @@ -356,18 +356,43 @@ function isFLE2ReplicationEnabled() { } /** - * Assert a field is an indexed encrypted field + * Assert a field is an indexed encrypted field. That includes both + * equality and range * * @param {BinData} value bindata value */ function assertIsIndexedEncryptedField(value) { assert(value instanceof BinData, "Expected BinData, found: " + value); assert.eq(value.subtype(), 6, "Expected Encrypted bindata: " + value); + assert(value.hex().startsWith("07") || value.hex().startsWith("09"), + "Expected subtype 7 or 9 but found the wrong type: " + value.hex()); +} + +/** + * Assert a field is an equality indexed encrypted field + * + * @param {BinData} value bindata value + */ +function assertIsEqualityIndexedEncryptedField(value) { + assert(value instanceof BinData, "Expected BinData, found: " + value); + assert.eq(value.subtype(), 6, "Expected Encrypted bindata: " + value); assert(value.hex().startsWith("07"), "Expected subtype 7 but found the wrong type: " + value.hex()); } /** + * Assert a field is a range indexed encrypted field + * + * @param {BinData} value bindata value + */ +function assertIsRangeIndexedEncryptedField(value) { + assert(value instanceof BinData, "Expected BinData, found: " + value); + assert.eq(value.subtype(), 6, "Expected Encrypted bindata: " + value); + assert(value.hex().startsWith("09"), + "Expected subtype 9 but found the wrong type: " + value.hex()); +} + +/** * Assert a field is an unindexed encrypted field * * @param {BinData} value bindata value diff --git a/src/mongo/crypto/encryption_fields.idl b/src/mongo/crypto/encryption_fields.idl index 5b989d09f54..969587502f0 100644 --- a/src/mongo/crypto/encryption_fields.idl +++ b/src/mongo/crypto/encryption_fields.idl @@ -91,7 +91,7 @@ structs: type: exactInt64 optional: true stability: unstable - validator: { gte: 0, lte: 3 } + validator: { gte: 1, lte: 4 } EncryptedField: description: "Information about encrypted fields" diff --git a/src/mongo/crypto/fle_crypto.cpp b/src/mongo/crypto/fle_crypto.cpp index 7c508dfd459..814d99bf95e 100644 --- a/src/mongo/crypto/fle_crypto.cpp +++ b/src/mongo/crypto/fle_crypto.cpp @@ -33,6 +33,7 @@ #include <cmath> #include <cstdint> #include <functional> +#include <iomanip> #include <iterator> #include <limits> #include <memory> @@ -698,30 +699,71 @@ std::vector<std::string> getMinCover(const FLE2RangeSpec& spec, uint8_t sparsity return edges; } -// Note: Does not return the leaf node -std::vector<std::string> getEdges(BSONElement element, - BSONElement lowerBound, - BSONElement upperBound, - uint8_t sparsity) { - // TODO SERVER-69013 - return {"1", "01", "001"}; +std::unique_ptr<Edges> getEdges(BSONElement element, + BSONElement lowerBound, + BSONElement upperBound, + int sparsity) { + switch (element.type()) { + case BSONType::NumberInt: + uassert( + 6775501, "lower bound must be integer", lowerBound.type() == BSONType::NumberInt); + uassert( + 6775502, "upper bound must be integer", upperBound.type() == BSONType::NumberInt); + return getEdgesInt32(element.Int(), lowerBound.Int(), upperBound.Int(), sparsity); + + case BSONType::NumberLong: + uassert( + 6775503, "lower bound must be long int", lowerBound.type() == BSONType::NumberLong); + uassert( + 6775504, "upper bound must be long int", upperBound.type() == BSONType::NumberLong); + return getEdgesInt64(element.Long(), lowerBound.Long(), upperBound.Long(), sparsity); + + case BSONType::Date: + uassert(6775505, "lower bound must be date", lowerBound.type() == BSONType::Date); + uassert(6775506, "upper bound must be date", upperBound.type() == BSONType::Date); + return getEdgesInt64(element.Date().asInt64(), + lowerBound.Date().asInt64(), + upperBound.Date().asInt64(), + sparsity); + + case BSONType::NumberDouble: + uassert( + 6775507, "lower bound must be double", lowerBound.type() == BSONType::NumberDouble); + uassert( + 6775508, "upper bound must be double", upperBound.type() == BSONType::NumberDouble); + return getEdgesDouble( + element.Double(), lowerBound.Double(), upperBound.Double(), sparsity); + + case BSONType::NumberDecimal: + uassert(6775509, + "lower bound must be decimal", + lowerBound.type() == BSONType::NumberDecimal); + uassert(6775510, + "upper bound must be decimal", + upperBound.type() == BSONType::NumberDecimal); + uasserted(ErrorCodes::BadValue, "decimal not supported atm"); + + default: + uassert(6775500, "must use supported FLE2 range type", false); + } } std::vector<EdgeTokenSet> getEdgeTokenSet(BSONElement element, BSONElement lowerBound, BSONElement upperBound, - uint8_t sparsity, + int sparsity, uint64_t contentionFactor, const EDCToken& edcToken, const ESCToken& escToken, const ECCToken& eccToken, const ECOCToken& ecocToken) { - auto edges = getEdges(element, lowerBound, upperBound, sparsity); + const auto edges = getEdges(element, lowerBound, upperBound, sparsity); + const auto edgesList = edges->get(); std::vector<EdgeTokenSet> tokens; - for (const auto& edge : edges) { - ConstDataRange cdr(edge.data(), edge.size()); + for (const auto& edge : edgesList) { + ConstDataRange cdr(edge.rawData(), edge.size()); EDCDerivedFromDataToken edcDatakey = FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, cdr); @@ -1190,7 +1232,6 @@ void convertServerPayload(ConstDataRange cdr, ConstVectorIteratorPair<EDCServerPayloadInfo>& it, BSONObjBuilder* builder, StringData fieldPath) { - auto [encryptedTypeBinding, subCdr] = fromEncryptedConstDataRange(cdr); if (encryptedTypeBinding == EncryptedBinDataType::kFLE2FindEqualityPayload || encryptedTypeBinding == EncryptedBinDataType::kFLE2FindRangePayload) { @@ -1203,11 +1244,11 @@ void convertServerPayload(ConstDataRange cdr, } uassert(6373505, "Unexpected end of iterator", it.it != it.end); - auto payload = *(it.it); + const auto payload = it.it; // TODO - validate field is actually indexed in the schema? - if (payload.payload.getEdgeTokenSet().has_value()) { - FLE2IndexedRangeEncryptedValue sp(payload.payload, payload.counts); + if (payload->payload.getEdgeTokenSet().has_value()) { + FLE2IndexedRangeEncryptedValue sp(payload->payload, payload->counts); uassert(6775311, str::stream() << "Type '" << typeName(sp.bsonType) @@ -1216,7 +1257,7 @@ void convertServerPayload(ConstDataRange cdr, auto swEncrypted = sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload.payload.getServerEncryptionToken())); + payload->payload.getServerEncryptionToken())); uassertStatusOK(swEncrypted); toEncryptedBinData(fieldPath, EncryptedBinDataType::kFLE2RangeIndexedValue, @@ -1228,28 +1269,28 @@ void convertServerPayload(ConstDataRange cdr, pTags->push_back({tag}); } - return; - } + } else { + dassert(payload->counts.size() == 1); - dassert(payload.counts.size() == 1); + FLE2IndexedEqualityEncryptedValue sp(payload->payload, payload->counts[0]); - FLE2IndexedEqualityEncryptedValue sp(payload.payload, payload.counts[0]); + uassert(6373506, + str::stream() << "Type '" << typeName(sp.bsonType) + << "' is not a valid type for Queryable Encryption Equality", + isFLE2EqualityIndexedSupportedType(sp.bsonType)); + + auto swEncrypted = + sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( + payload->payload.getServerEncryptionToken())); + uassertStatusOK(swEncrypted); + toEncryptedBinData(fieldPath, + EncryptedBinDataType::kFLE2EqualityIndexedValue, + ConstDataRange(swEncrypted.getValue()), + builder); + + pTags->push_back({EDCServerCollection::generateTag(*payload)}); + } - uassert(6373506, - str::stream() << "Type '" << typeName(sp.bsonType) - << "' is not a valid type for Queryable Encryption Equality", - isFLE2EqualityIndexedSupportedType(sp.bsonType)); - - auto swEncrypted = - sp.serialize(FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload.payload.getServerEncryptionToken())); - uassertStatusOK(swEncrypted); - toEncryptedBinData(fieldPath, - EncryptedBinDataType::kFLE2EqualityIndexedValue, - ConstDataRange(swEncrypted.getValue()), - builder); - - pTags->push_back({EDCServerCollection::generateTag(payload)}); } else if (encryptedTypeBinding == EncryptedBinDataType::kFLE2UnindexedEncryptedValue) { builder->appendBinData(fieldPath, cdr.length(), BinDataType::Encrypt, cdr.data()); return; @@ -2353,29 +2394,21 @@ std::pair<BSONType, std::vector<uint8_t>> FLE2UnindexedEncryptedValue::deseriali } std::vector<FLEEdgeToken> toFLEEdgeTokenSet(const FLE2InsertUpdatePayload& payload) { + const auto ets = payload.getEdgeTokenSet().get(); std::vector<FLEEdgeToken> tokens; - tokens.reserve(payload.getEdgeTokenSet().get().size() + 1); - - FLEEdgeToken leaf; - leaf.edc = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEdcDerivedToken()); - leaf.esc = FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEscDerivedToken()); - leaf.ecc = FLETokenFromCDR<FLETokenType::ECCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEccDerivedToken()); - tokens.push_back(leaf); - - for (const auto& ets : payload.getEdgeTokenSet().get()) { + tokens.reserve(ets.size()); + + for (const auto& et : ets) { FLEEdgeToken edgeToken; edgeToken.edc = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataTokenAndContentionFactorToken>( - ets.getEdcDerivedToken()); + et.getEdcDerivedToken()); edgeToken.esc = FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( - ets.getEscDerivedToken()); + et.getEscDerivedToken()); edgeToken.ecc = FLETokenFromCDR<FLETokenType::ECCDerivedFromDataTokenAndContentionFactorToken>( - ets.getEccDerivedToken()); + et.getEccDerivedToken()); tokens.push_back(edgeToken); } @@ -2571,9 +2604,9 @@ StatusWith<std::vector<uint8_t>> FLE2IndexedRangeEncryptedValue::serialize( } -ESCDerivedFromDataTokenAndContentionFactorToken EDCServerPayloadInfo::getESCToken() const { - return FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>( - payload.getEscDerivedToken()); +ESCDerivedFromDataTokenAndContentionFactorToken EDCServerPayloadInfo::getESCToken( + ConstDataRange cdr) { + return FLETokenFromCDR<FLETokenType::ESCDerivedFromDataTokenAndContentionFactorToken>(cdr); } void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, diff --git a/src/mongo/crypto/fle_crypto.h b/src/mongo/crypto/fle_crypto.h index 65178dcedb1..15617f2460e 100644 --- a/src/mongo/crypto/fle_crypto.h +++ b/src/mongo/crypto/fle_crypto.h @@ -1033,9 +1033,8 @@ struct FLE2IndexedRangeEncryptedValue { std::vector<uint8_t> clientEncryptedValue; }; - struct EDCServerPayloadInfo { - ESCDerivedFromDataTokenAndContentionFactorToken getESCToken() const; + static ESCDerivedFromDataTokenAndContentionFactorToken getESCToken(ConstDataRange cdr); FLE2InsertUpdatePayload payload; std::string fieldPathName; diff --git a/src/mongo/crypto/fle_crypto_test.cpp b/src/mongo/crypto/fle_crypto_test.cpp index da682bf8b70..705aee8da6c 100644 --- a/src/mongo/crypto/fle_crypto_test.cpp +++ b/src/mongo/crypto/fle_crypto_test.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include "mongo/base/error_codes.h" #include "mongo/platform/basic.h" #include "mongo/crypto/fle_crypto.h" @@ -717,10 +718,38 @@ std::vector<char> generatePlaceholder( FLE2RangeInsertSpec insertSpec; // Set a default lower and upper bound - auto lowerDoc = BSON("lb" << 0); + BSONObj lowerDoc, upperDoc; + switch (value.type()) { + case BSONType::NumberInt: + lowerDoc = BSON("lb" << 0); + upperDoc = BSON("ub" << 1234567); + break; + case BSONType::NumberLong: + lowerDoc = BSON("lb" << 0LL); + upperDoc = BSON("ub" << 1234567890123456789LL); + break; + case BSONType::NumberDouble: + lowerDoc = BSON("lb" << 0.0); + upperDoc = BSON("ub" << 1234567890123456789.0); + break; + case BSONType::Date: + lowerDoc = BSON("lb" << Date_t::fromMillisSinceEpoch(0)); + upperDoc = BSON("ub" << Date_t::fromMillisSinceEpoch(1234567890123456789LL)); + break; + case BSONType::NumberDecimal: + lowerDoc = BSON("lb" << Decimal128(0)); + upperDoc = BSON("ub" << Decimal128(1234567890123456789LL)); + break; + default: + LOGV2_WARNING(6775520, + "Invalid type for range algo", + "algo"_attr = algorithm, + "type"_attr = value.type()); + lowerDoc = BSON("lb" << 0); + upperDoc = BSON("ub" << 1234567); + break; + } insertSpec.setLowerBound(lowerDoc.firstElement()); - auto upperDoc = BSON("ub" << 1234567890123456789LL); - insertSpec.setUpperBound(upperDoc.firstElement()); insertSpec.setValue(value); auto specDoc = BSON("s" << insertSpec.toBSON()); @@ -738,7 +767,7 @@ std::vector<char> generatePlaceholder( } else if (operation == Operation::kInsert) { ep.setValue(IDLAnyType(specDoc.firstElement())); } - ep.setSparsity(0); + ep.setSparsity(1); } else { ep.setValue(value); } @@ -771,9 +800,9 @@ BSONObj encryptDocument(BSONObj obj, for (size_t i = 0; i < payload.payload.getEdgeTokenSet()->size(); i++) { payload.counts.push_back(1); } + } else { + payload.counts.push_back(1); } - - payload.counts.push_back(1); } // Finalize document for insert @@ -975,7 +1004,8 @@ TEST(FLE_EDC, Range_Allowed_Types) { const std::vector<std::pair<BSONObj, BSONType>> rangeAllowedObjects{ {BSON("sample" << 123.456), NumberDouble}, - {BSON("sample" << Decimal128()), NumberDecimal}, + // TODO SERVER-68542 remove the commented line + // {BSON("sample" << Decimal128()), NumberDecimal}, {BSON("sample" << 123456), NumberInt}, {BSON("sample" << 12345678901234567LL), NumberLong}, {BSON("sample" << Date_t::fromMillisSinceEpoch(12345)), Date}, @@ -1333,7 +1363,7 @@ TEST(FLE_EDC, ServerSide_Range_Payloads) { iupayload.setEdgeTokenSet(tokens); - FLE2IndexedRangeEncryptedValue serverPayload(iupayload, {123456, 123456, 123456}); + FLE2IndexedRangeEncryptedValue serverPayload(iupayload, {123456, 123456}); auto swBuf = serverPayload.serialize(serverEncryptToken); ASSERT_OK(swBuf.getStatus()); @@ -1343,7 +1373,7 @@ TEST(FLE_EDC, ServerSide_Range_Payloads) { ASSERT_OK(swServerPayload.getStatus()); auto sp = swServerPayload.getValue(); - ASSERT_EQ(sp.tokens.size(), 3); + ASSERT_EQ(sp.tokens.size(), 2); for (size_t i = 0; i < sp.tokens.size(); i++) { auto ets = sp.tokens[i]; auto rhs = serverPayload.tokens[i]; @@ -1747,12 +1777,13 @@ TEST(EDC, ValidateDocument) { auto buf = generatePlaceholder(element, Operation::kInsert); builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); } + + BSONObjBuilder sub(builder.subobjStart("nested")); { auto doc = BSON("a" << "top secret"); auto element = doc.firstElement(); - BSONObjBuilder sub(builder.subobjStart("nested")); auto buf = generatePlaceholder( element, Operation::kInsert, Fle2AlgorithmInt::kEquality, indexKey2Id); builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); @@ -1762,12 +1793,13 @@ TEST(EDC, ValidateDocument) { << "bottom secret"); auto element = doc.firstElement(); - BSONObjBuilder sub(builder.subobjStart("nested")); auto buf = generatePlaceholder(element, Operation::kInsert, Fle2AlgorithmInt::kUnindexed); builder.appendBinData("notindexed", buf.size(), BinDataType::Encrypt, buf.data()); } + sub.done(); - auto finalDoc = encryptDocument(builder.obj(), &keyVault, &efc); + auto doc1 = builder.obj(); + auto finalDoc = encryptDocument(doc1, &keyVault, &efc); // Positive - Encrypted Doc FLEClientCrypto::validateDocument(finalDoc, efc, &keyVault); diff --git a/src/mongo/crypto/fle_field_schema.idl b/src/mongo/crypto/fle_field_schema.idl index 0e5be537a74..fa83ea2f369 100644 --- a/src/mongo/crypto/fle_field_schema.idl +++ b/src/mongo/crypto/fle_field_schema.idl @@ -166,7 +166,7 @@ structs: type: int cpp_name: sparsity optional: true - validator: { gte: 0, lte: 3 } + validator: { gte: 1, lte: 4 } EdgeTokenSet: description: "Payload of an indexed field to insert or update" diff --git a/src/mongo/crypto/fle_fields_util.cpp b/src/mongo/crypto/fle_fields_util.cpp index 4905d42176b..131e0a59ab1 100644 --- a/src/mongo/crypto/fle_fields_util.cpp +++ b/src/mongo/crypto/fle_fields_util.cpp @@ -65,10 +65,12 @@ void validateIDLFLE2RangeSpec(const FLE2RangeSpec* placeholder) { auto min = placeholder->getMin().getElement(); auto max = placeholder->getMax().getElement(); uassert(6833400, - str::stream() << "Minimum element in a range must be numeric, not: " << min.type(), - min.isNumber() || min.type() == BSONType::MinKey); + str::stream() << "Minimum element in a range must be numeric or date, not: " + << min.type(), + min.isNumber() || min.type() == BSONType::MinKey || min.type() == BSONType::Date); uassert(6833401, - str::stream() << "Maximum element in a range must be numeric, not: " << max.type(), - max.isNumber() || max.type() == BSONType::MaxKey); + str::stream() << "Maximum element in a range must be numeric or date, not: " + << max.type(), + max.isNumber() || max.type() == BSONType::MaxKey || min.type() == BSONType::Date); } } // namespace mongo diff --git a/src/mongo/db/catalog/collection_options_validation.cpp b/src/mongo/db/catalog/collection_options_validation.cpp index 840e80c4922..9dc4535c41c 100644 --- a/src/mongo/db/catalog/collection_options_validation.cpp +++ b/src/mongo/db/catalog/collection_options_validation.cpp @@ -143,8 +143,8 @@ EncryptedFieldConfig processAndValidateEncryptedFields(EncryptedFieldConfig conf query.getMax().has_value()); uassert(6775214, - "The field 'sparsity' must be between 0 and 3", - query.getSparsity().value() >= 1 && query.getSparsity().value() <= 3); + "The field 'sparsity' must be between 1 and 4", + query.getSparsity().value() >= 1 && query.getSparsity().value() <= 4); // Check type compatibility switch (type) { diff --git a/src/mongo/db/fle_crud.cpp b/src/mongo/db/fle_crud.cpp index 7a6ed7eb09f..80a3e07aa24 100644 --- a/src/mongo/db/fle_crud.cpp +++ b/src/mongo/db/fle_crud.cpp @@ -28,10 +28,10 @@ */ -#include "mongo/db/fle_crud.h" - +#include <algorithm> #include <memory> +#include "mongo/base/data_range.h" #include "mongo/bson/bsonelement.h" #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" @@ -41,6 +41,7 @@ #include "mongo/crypto/fle_crypto.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/dbdirectclient.h" +#include "mongo/db/fle_crud.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_time_tracker.h" #include "mongo/db/ops/write_ops_gen.h" @@ -493,68 +494,83 @@ void processFieldsForInsert(FLEQueryInterface* queryImpl, for (auto& payload : serverPayload) { - auto escToken = payload.getESCToken(); - auto tagToken = FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedTagToken(escToken); - auto valueToken = - FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedValueToken(escToken); - payload.counts.clear(); + const auto insertTokens = [&](ConstDataRange encryptedTokens, + ConstDataRange escDerivedToken) { + int position = 1; + int count = 1; - int position = 1; - int count = 1; - auto alpha = ESCCollection::emuBinary(reader, tagToken, valueToken); + auto escToken = EDCServerPayloadInfo::getESCToken(escDerivedToken); + auto tagToken = + FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedTagToken(escToken); + auto valueToken = + FLETwiceDerivedTokenGenerator::generateESCTwiceDerivedValueToken(escToken); - if (alpha.has_value() && alpha.value() == 0) { - position = 1; - count = 1; - } else if (!alpha.has_value()) { - auto block = ESCCollection::generateId(tagToken, boost::none); + auto alpha = ESCCollection::emuBinary(reader, tagToken, valueToken); - auto r_esc = reader.getById(block); - uassert(6371203, "ESC document not found", !r_esc.isEmpty()); + if (alpha.has_value() && alpha.value() == 0) { + position = 1; + count = 1; + } else if (!alpha.has_value()) { + auto block = ESCCollection::generateId(tagToken, boost::none); - auto escNullDoc = - uassertStatusOK(ESCCollection::decryptNullDocument(valueToken, r_esc)); + auto r_esc = reader.getById(block); + uassert(6371203, "ESC document not found", !r_esc.isEmpty()); - position = escNullDoc.position + 2; - count = escNullDoc.count + 1; - } else { - auto block = ESCCollection::generateId(tagToken, alpha); + auto escNullDoc = + uassertStatusOK(ESCCollection::decryptNullDocument(valueToken, r_esc)); + + position = escNullDoc.position + 2; + count = escNullDoc.count + 1; + } else { + auto block = ESCCollection::generateId(tagToken, alpha); - auto r_esc = reader.getById(block); - uassert(6371204, "ESC document not found", !r_esc.isEmpty()); + auto r_esc = reader.getById(block); + uassert(6371204, "ESC document not found", !r_esc.isEmpty()); - auto escDoc = uassertStatusOK(ESCCollection::decryptDocument(valueToken, r_esc)); + auto escDoc = uassertStatusOK(ESCCollection::decryptDocument(valueToken, r_esc)); - position = alpha.value() + 1; - count = escDoc.count + 1; + position = alpha.value() + 1; + count = escDoc.count + 1; - if (escDoc.compactionPlaceholder) { - uassertStatusOK(Status(ErrorCodes::FLECompactionPlaceholder, - "Found ESC contention placeholder")); + if (escDoc.compactionPlaceholder) { + uassertStatusOK(Status(ErrorCodes::FLECompactionPlaceholder, + "Found ESC contention placeholder")); + } } - } - payload.counts.push_back(count); + payload.counts.push_back(count); - auto escInsertReply = uassertStatusOK(queryImpl->insertDocument( - nssEsc, - ESCCollection::generateInsertDocument(tagToken, valueToken, position, count), - pStmtId, - true)); - checkWriteErrors(escInsertReply); + auto escInsertReply = uassertStatusOK(queryImpl->insertDocument( + nssEsc, + ESCCollection::generateInsertDocument(tagToken, valueToken, position, count), + pStmtId, + true)); + checkWriteErrors(escInsertReply); - NamespaceString nssEcoc(edcNss.db(), efc.getEcocCollection().value()); + NamespaceString nssEcoc(edcNss.db(), efc.getEcocCollection().value()); - // TODO - should we make this a batch of ECOC updates? - auto ecocInsertReply = uassertStatusOK(queryImpl->insertDocument( - nssEcoc, - ECOCCollection::generateDocument(payload.fieldPathName, - payload.payload.getEncryptedTokens()), - pStmtId, - false, - bypassDocumentValidation)); - checkWriteErrors(ecocInsertReply); + // TODO - should we make this a batch of ECOC updates? + auto ecocInsertReply = uassertStatusOK(queryImpl->insertDocument( + nssEcoc, + ECOCCollection::generateDocument(payload.fieldPathName, encryptedTokens), + pStmtId, + false, + bypassDocumentValidation)); + checkWriteErrors(ecocInsertReply); + }; + + payload.counts.clear(); + const bool isRangePayload = payload.payload.getEdgeTokenSet().has_value(); + if (isRangePayload) { + const auto ets = payload.payload.getEdgeTokenSet().get(); + for (size_t i = 0; i < ets.size(); ++i) { + insertTokens(ets[i].getEncryptedTokens(), ets[i].getEscDerivedToken()); + } + } else { + insertTokens(payload.payload.getEncryptedTokens(), + payload.payload.getEscDerivedToken()); + } } } diff --git a/src/mongo/db/matcher/expression_type.h b/src/mongo/db/matcher/expression_type.h index 6bdd8073736..dfc18407a3b 100644 --- a/src/mongo/db/matcher/expression_type.h +++ b/src/mongo/db/matcher/expression_type.h @@ -411,6 +411,7 @@ public: EncryptedBinDataType subTypeByte = static_cast<EncryptedBinDataType>(binData[0]); switch (subTypeByte) { case EncryptedBinDataType::kFLE2EqualityIndexedValue: + case EncryptedBinDataType::kFLE2RangeIndexedValue: case EncryptedBinDataType::kFLE2UnindexedEncryptedValue: { // Verify the type of the encrypted data. if (typeSet().isEmpty()) { diff --git a/src/mongo/db/matcher/expression_type_test.cpp b/src/mongo/db/matcher/expression_type_test.cpp index bf7d40e836a..f927c787e1f 100644 --- a/src/mongo/db/matcher/expression_type_test.cpp +++ b/src/mongo/db/matcher/expression_type_test.cpp @@ -386,6 +386,7 @@ TEST(InternalSchemaBinDataFLE2EncryptedTypeTest, MatchesOnlyFLE2ServerSubtypes) reinterpret_cast<const void*>(&blob), sizeof(FleBlobHeader), BinDataType::Encrypt); if (i == static_cast<uint8_t>(EncryptedBinDataType::kFLE2EqualityIndexedValue) || + i == static_cast<uint8_t>(EncryptedBinDataType::kFLE2RangeIndexedValue) || i == static_cast<uint8_t>(EncryptedBinDataType::kFLE2UnindexedEncryptedValue)) { ASSERT_TRUE(expr.matchesBSON(BSON("a" << binData))); } else { diff --git a/src/mongo/shell/shell_options.cpp b/src/mongo/shell/shell_options.cpp index e23f89ff7dc..69bbb071493 100644 --- a/src/mongo/shell/shell_options.cpp +++ b/src/mongo/shell/shell_options.cpp @@ -67,6 +67,7 @@ const std::set<std::string> kSetShellParameterAllowlist = { "awsEC2InstanceMetadataUrl", "awsECSInstanceMetadataUrl", "disabledSecureAllocatorDomains", + "featureFlagFLE2Range", "newLineAfterPasswordPromptForTest", "ocspClientHttpTimeoutSecs", "ocspEnabled", |