diff options
| author | Eric Cox <eric.cox@mongodb.com> | 2022-06-22 14:21:07 +0000 |
|---|---|---|
| committer | Eric Cox <eric.cox@mongodb.com> | 2022-06-22 14:21:07 +0000 |
| commit | 44385b89a690f1feda55d8e888a08f33eab0a020 (patch) | |
| tree | 6ec320d4c7de3e7ff00231aec91106f844701cc3 | |
| parent | e24af2270e6bc1d435845c8fdd02a1eb24155da2 (diff) | |
| download | mongo-44385b89a690f1feda55d8e888a08f33eab0a020.tar.gz | |
Revert "Merge branch 'v6.0' of github.com:10gen/mongo into v6.0"
This reverts commit e24af2270e6bc1d435845c8fdd02a1eb24155da2, reversing
changes made to ecf315a92efb6283a4c8f3fd079dd81398563fa6.
94 files changed, 612 insertions, 2484 deletions
diff --git a/buildscripts/idl/idl/bson.py b/buildscripts/idl/idl/bson.py index 8216b5d743d..3f8e5190f9d 100644 --- a/buildscripts/idl/idl/bson.py +++ b/buildscripts/idl/idl/bson.py @@ -72,7 +72,6 @@ _BINDATA_SUBTYPE = { # }, "uuid": {'scalar': True, 'bindata_enum': 'newUUID'}, "md5": {'scalar': True, 'bindata_enum': 'MD5Type'}, - "encrypt": {'scalar': True, 'bindata_enum': 'Encrypt'}, } diff --git a/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml b/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml index 22bf09a9f5c..e05f4ed1a0f 100644 --- a/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml +++ b/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml @@ -9,7 +9,3 @@ # and TenantDatabase constructors. - featureFlagRequireTenantID - featureFlagSbePlanCache -# This flag exists to help users in managed environments that upgraded to 6.0 before 6.0.0-rc8 was -# released create the transactions collection index and is only meant to be enabled adhoc, so only -# its targeted tests should enable it. -- featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp diff --git a/buildscripts/resmokeconfig/suites/change_streams_downgrade.yml b/buildscripts/resmokeconfig/suites/change_streams_downgrade.yml index bb56d4e0cc2..e172efdc0e2 100644 --- a/buildscripts/resmokeconfig/suites/change_streams_downgrade.yml +++ b/buildscripts/resmokeconfig/suites/change_streams_downgrade.yml @@ -149,6 +149,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml b/buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml deleted file mode 100644 index 4e6002ae7d2..00000000000 --- a/buildscripts/resmokeconfig/suites/fle2_high_cardinality.yml +++ /dev/null @@ -1,32 +0,0 @@ -test_kind: js_test -selector: - roots: - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/*.js - - src/mongo/db/modules/*/jstests/fle2/query/*.js - exclude_with_any_tags: - # Not compatible with tests the expect fle to always using $in in queries, - # i.e. verify explain output - - requires_fle_in_always - -executor: - archive: - hooks: - - ValidateCollections - config: - shell_options: - eval: "testingReplication = true; testingFLESharding = false;" - hooks: - # We don't execute dbHash or oplog consistency checks since there is only a single replica set - # node. - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ReplicaSetFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - internalQueryFLEAlwaysUseHighCardinalityMode: 1 - # Use a 2-node replica set. - num_nodes: 2 diff --git a/buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml b/buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml deleted file mode 100644 index 33a3d4e5c1a..00000000000 --- a/buildscripts/resmokeconfig/suites/fle2_sharding_high_cardinality.yml +++ /dev/null @@ -1,37 +0,0 @@ -test_kind: js_test -selector: - roots: - - jstests/fle2/*.js - - src/mongo/db/modules/*/jstests/fle2/*.js - - src/mongo/db/modules/*/jstests/fle2/query/*.js - exclude_with_any_tags: - # Not compatible with tests the expect fle to always using $in in queries, - # i.e. verify explain output - - requires_fle_in_always - -executor: - archive: - hooks: - - CheckReplDBHash - - ValidateCollections - config: - shell_options: - eval: "testingReplication = false; testingFLESharding = true;" - hooks: - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - mongos_options: - set_parameters: - enableTestCommands: 1 - internalQueryFLEAlwaysUseHighCardinalityMode: 1 - mongod_options: - set_parameters: - enableTestCommands: 1 - internalQueryFLEAlwaysUseHighCardinalityMode: 1 - num_rs_nodes_per_shard: 2 - enable_sharding: - - test diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml index d44172752fb..0194d0b5a60 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml index 2633fe3b32e..47213b4a54c 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml index ffa1b1c514f..348d549df2f 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml index fd4a882859e..4f1d19d2d48 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml index b3e53d6b280..d633a55e6f7 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml @@ -57,6 +57,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded # queries with a limit or for distinct commands. diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml index 2249df645c1..887ef290c4c 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml @@ -57,6 +57,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded # queries with a limit or for distinct commands. diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml index cc966fdf43b..b4dae1490eb 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml @@ -57,6 +57,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded # queries with a limit or for distinct commands. diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml index fbf136b1b33..495de2e43e8 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml @@ -57,6 +57,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded # queries with a limit or for distinct commands. diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml index dae009af297..77ee0700640 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml @@ -13,6 +13,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # These tests verify that an expected number of update operations were tracked in the server # status metrics, but the logical session cache refresh causes additional updates to be recorded. diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml index 182b59965a1..c3ccea25a2d 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml @@ -13,6 +13,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # These tests verify that an expected number of update operations were tracked in the server # status metrics, but the logical session cache refresh causes additional updates to be recorded. diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml index 8b4db08b8bf..110804994a4 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml @@ -13,6 +13,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # These tests verify that an expected number of update operations were tracked in the server # status metrics, but the logical session cache refresh causes additional updates to be recorded. diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml index 62886996b51..962c41c03f3 100644 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml @@ -13,6 +13,7 @@ selector: # can be triggered deterministically. - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # These tests verify that an expected number of update operations were tracked in the server # status metrics, but the logical session cache refresh causes additional updates to be recorded. diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml index f215abee05a..07e839d9b77 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml @@ -151,6 +151,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml index 8e21d6b8999..26d4fc14d97 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -146,6 +146,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml index 511123c8ce5..84d46018091 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml @@ -147,6 +147,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml index b30c65e8e4c..ccdf9fcec1b 100644 --- a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml +++ b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml @@ -159,6 +159,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml index b6316ac186a..39549ffe941 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml @@ -96,6 +96,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $indexStats diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml index bc58d35542c..780cf31a717 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -87,6 +87,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml index 87e35d4f909..8edf927513e 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml @@ -86,6 +86,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml index daff4e411c8..2fad03caff8 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml @@ -84,6 +84,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $collStats diff --git a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml index 9c69c04eef2..2c0dd75a63c 100644 --- a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml @@ -120,6 +120,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $indexStats diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml index c3c81736ea2..9413b35ee3e 100644 --- a/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/tenant_migration_multi_stmt_txn_jscore_passthrough.yml @@ -111,6 +111,7 @@ selector: # $listSessions - jstests/core/list_all_local_sessions.js - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js - jstests/core/list_sessions.js # $indexStats diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 6c7c873bf09..640b0e54901 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1229,8 +1229,7 @@ buildvariants: - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough - # TODO: SERVER-66945 Re-enable ESE on enterprise macos - # - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore !.ssl + - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore !.ssl - name: .jscore .common !.decimal !.sharding - name: .logical_session_cache .one_sec - name: mqlrun diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index b0db6165d9b..d058492d86a 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -6003,20 +6003,6 @@ tasks: - func: "run tests" - <<: *task_template - name: fle2_high_cardinality - tags: ["encrypt"] - commands: - - func: "do setup" - - func: "run tests" - -- <<: *task_template - name: fle2_sharding_high_cardinality - tags: ["encrypt"] - commands: - - func: "do setup" - - func: "run tests" - -- <<: *task_template name: ocsp tags: ["ssl", "encrypt", "ocsp"] commands: diff --git a/etc/evergreen_yml_components/variants/misc_release.yml b/etc/evergreen_yml_components/variants/misc_release.yml index 7d01683ecfb..14f60d11b81 100644 --- a/etc/evergreen_yml_components/variants/misc_release.yml +++ b/etc/evergreen_yml_components/variants/misc_release.yml @@ -1862,8 +1862,7 @@ buildvariants: - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough - # TODO: SERVER-66945 Re-enable ESE on enterprise macos - # - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore + - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore - name: .jscore .common !.decimal !.sharding - name: .jstestfuzz .common - name: .logical_session_cache .one_sec diff --git a/jstests/aggregation/bugs/server66418.js b/jstests/aggregation/bugs/server66418.js deleted file mode 100644 index 9b8c960282a..00000000000 --- a/jstests/aggregation/bugs/server66418.js +++ /dev/null @@ -1,38 +0,0 @@ -// SERVER-66418 -// Bad projection created during dependency analysis due to string order assumption -(function() { -"use strict"; - -const coll = db[jsTest.name()]; -coll.drop(); - -coll.save({ - _id: 1, - type: 'PRODUCT', - status: 'VALID', - locale: { - en: 'INSTRUMENT PANEL', - es: 'INSTRUMENTOS DEL CUADRO', - fr: 'INSTRUMENT TABLEAU DE BORD', - } -}); - -// before SERVER-66418, this incorrectly threw a PathCollision error -coll.aggregate([ - {"$match": {"_id": 1}}, - {"$sort": {"_id": 1}}, - { - "$project": { - "designation": { - "$switch": { - "branches": [{ - "case": {"$eq": ["$type", "PRODUCT"]}, - "then": {"$ifNull": ["$locale.en-GB.name", "$locale.en.name"]} - }], - "default": {"$ifNull": ["$locale.en-GB", "$locale.en"]} - } - } - } - } -]); -})(); diff --git a/jstests/aggregation/sources/densify/libs/parse_util.js b/jstests/aggregation/sources/densify/libs/parse_util.js index d2e6ba4e489..7ccedefd81d 100644 --- a/jstests/aggregation/sources/densify/libs/parse_util.js +++ b/jstests/aggregation/sources/densify/libs/parse_util.js @@ -139,20 +139,6 @@ let parseUtil = (function(db, coll, stageName, options = {}) { }), 5733402, "a bounding array must be an ascending array of either two dates or two numbers"); - // Non-whole number step with date bounds - assert.commandFailedWithCode( - run({ - [stageName]: { - field: "a", - range: { - step: 1.1, - bounds: [new ISODate("2020-01-01"), new ISODate("2020-01-03")], - unit: "second" - } - } - }), - 6586400, - "The step parameter in a range satement must be a whole number when densifying a date range"); // Positive test cases assert.commandWorked(run({[stageName]: {field: "a", range: {step: 1.0, bounds: [1, 2]}}})); diff --git a/jstests/core/capped_resize.js b/jstests/core/capped_resize.js index 06baab6b21e..38cc7abd9a0 100644 --- a/jstests/core/capped_resize.js +++ b/jstests/core/capped_resize.js @@ -83,14 +83,6 @@ let verifyLimitUpdate = function(updates) { assert.eq(stats.count, initialDocSize); assert.lte(stats.size, maxSize); - // We used to not allow resizing the size of a capped collection below 4096 bytes. This - // restriction was lifted in SERVER-67036. - // We should see a reduction in collection size and count relative to the previous test case. - verifyLimitUpdate({cappedSize: 256}); - stats = assert.commandWorked(cappedColl.stats()); - assert.lt(stats.count, initialDocSize); - assert.lt(stats.size, maxSize); - // We expect the resizing of a capped collection to fail when maxSize <= 0 and maxSize > // maxSizeCeiling. const negativeSize = -1 * maxSize; diff --git a/jstests/noPassthrough/list_local_sessions.js b/jstests/core/list_local_sessions.js index 60b73f7f6ad..bee5c084ca7 100644 --- a/jstests/noPassthrough/list_local_sessions.js +++ b/jstests/core/list_local_sessions.js @@ -8,20 +8,12 @@ // # Sessions are asynchronously flushed to disk, so a stepdown immediately after calling // # startSession may cause this test to fail to find the returned sessionId. // does_not_support_stepdowns, -// requires_sharding, // ] (function() { 'use strict'; -const st = new ShardingTest({ - shards: 1, - mongos: 1, - other: {mongosOptions: {setParameter: {disableLogicalSessionCacheRefresh: true}}} -}); - -const admin = st.s.getDB("admin"); - +const admin = db.getSiblingDB('admin'); function listLocalSessions() { return admin.aggregate([{'$listLocalSessions': {allUsers: false}}]); } @@ -31,7 +23,7 @@ let originalLogLevel = assert.commandWorked(admin.setLogLevel(1)).was.verbosity; try { // Start a new session and capture its sessionId. - const myid = assert.commandWorked(st.s.adminCommand({startSession: 1})).id.id; + const myid = assert.commandWorked(db.runCommand({startSession: 1})).id.id; assert(myid !== undefined); // Ensure that the cache now contains the session and is visible. @@ -88,6 +80,4 @@ try { } finally { admin.setLogLevel(originalLogLevel); } - -st.stop(); })(); diff --git a/jstests/core/timeseries/bucket_unpacking_with_sort.js b/jstests/core/timeseries/bucket_unpacking_with_sort.js index 029697be74a..3e92a9e2461 100644 --- a/jstests/core/timeseries/bucket_unpacking_with_sort.js +++ b/jstests/core/timeseries/bucket_unpacking_with_sort.js @@ -236,12 +236,7 @@ const runRewritesTest = (sortSpec, // changing out from under us. const bucketSpanMatch = { $match: { - $expr: { - $lte: [ - {$subtract: ["$control.max.t", "$control.min.t"]}, - {$const: NumberLong(3600000)} - ] - }, + $expr: {$lte: [{$subtract: ["$control.max.t", "$control.min.t"]}, {$const: 3600000}]}, } }; let foundMatch = findFirstMatch(optExplain); diff --git a/jstests/libs/parallelTester.js b/jstests/libs/parallelTester.js index c0a31e54b86..8974cf569dd 100644 --- a/jstests/libs/parallelTester.js +++ b/jstests/libs/parallelTester.js @@ -311,6 +311,7 @@ if (typeof _threadInject != "undefined") { // run in parallel, they could interfere with the cache and cause failures. parallelFilesDir + "/list_all_local_sessions.js", parallelFilesDir + "/list_all_sessions.js", + parallelFilesDir + "/list_local_sessions.js", parallelFilesDir + "/list_sessions.js", ]; var serialTests = makeKeys(serialTestsArr); diff --git a/jstests/multiVersion/internal_sessions_setfcv_wait_for_transaction_coordinator_cleanup.js b/jstests/multiVersion/internal_sessions_setfcv_wait_for_transaction_coordinator_cleanup.js index e9b0aea9005..bd56ea89f96 100644 --- a/jstests/multiVersion/internal_sessions_setfcv_wait_for_transaction_coordinator_cleanup.js +++ b/jstests/multiVersion/internal_sessions_setfcv_wait_for_transaction_coordinator_cleanup.js @@ -35,8 +35,6 @@ assert.commandWorked(st.s.adminCommand({split: ns, middle: {_id: 0}})); assert.commandWorked( st.s.adminCommand({moveChunk: ns, find: {_id: 0}, to: participant1.shardName})); -st.refreshCatalogCacheForNs(st.s, ns); - function runTestBasic(lsid) { jsTest.log("Test transaction coordinator documents are deleted before downgrade finishes " + "with lsid: " + tojson(lsid)); @@ -44,6 +42,9 @@ function runTestBasic(lsid) { // Upgrade fcv to make sure cluster is on the latestFCV before starting any transactions. assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: latestFCV})); + // Upgrading to lastestFCV clears the filtering metadata of all collections. + st.refreshCatalogCacheForNs(st.s, ns); + let commitTxnFp = configureFailPoint(coordinator, "hangBeforeCommitingTxn"); let deleteCoordinatorDocFp = configureFailPoint(coordinator, "hangBeforeDeletingCoordinatorDoc"); @@ -128,6 +129,9 @@ function runTestWithFailoverBeforeDocumentRemoval(lsid) { // Upgrade fcv to make sure cluster is on the latestFCV before starting any transactions. assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: latestFCV})); + // Upgrading to lastestFCV clears the filtering metadata of all collections. + st.refreshCatalogCacheForNs(st.s, ns); + let commitTxnFp = configureFailPoint(coordinator, "hangBeforeCommitingTxn"); let deleteCoordinatorDocFp = configureFailPoint(coordinator, "hangBeforeDeletingCoordinatorDoc"); diff --git a/jstests/multiVersion/internal_transactions_index_setFCV.js b/jstests/multiVersion/internal_transactions_index_setFCV.js index a3ce2ead44a..712e8249a54 100644 --- a/jstests/multiVersion/internal_transactions_index_setFCV.js +++ b/jstests/multiVersion/internal_transactions_index_setFCV.js @@ -41,8 +41,7 @@ function assertPartialIndexDoesNotExist(node) { * Verifies the partial index is dropped/created on FCV transitions and retryable writes work in all * FCVs. */ -function runTest( - setFCVConn, modifyIndexConns, verifyIndexConns, rst, alwaysCreateFeatureFlagEnabled) { +function runTest(setFCVConn, modifyIndexConns, verifyIndexConns, rst) { // Start at latest FCV which should have the index. assert.commandWorked(setFCVConn.adminCommand({setFeatureCompatibilityVersion: latestFCV})); verifyIndexConns.forEach(conn => { @@ -55,12 +54,8 @@ function runTest( assertPartialIndexDoesNotExist(conn); }); - assert.commandWorked(setFCVConn.getDB("foo").runCommand( - {insert: "bar", documents: [{x: 1}], lsid: {id: UUID()}, txnNumber: NumberLong(11)})); - if (rst) { - // On step up to primary the index should not be created. Note this tests the empty - // collection case when alwaysCreateFeatureFlagEnabled is true. + // On step up to primary the index should not be created. let primary = rst.getPrimary(); // Clear the collection so we'd try to create the index. @@ -74,21 +69,8 @@ function runTest( rst.awaitReplication(); verifyIndexConns.forEach(conn => { reconnect(conn); - if (alwaysCreateFeatureFlagEnabled) { - assertPartialIndexExists(conn); - } else { - assertPartialIndexDoesNotExist(conn); - } + assertPartialIndexDoesNotExist(conn); }); - - if (alwaysCreateFeatureFlagEnabled) { - // The test expects no index after this block, so remove it. - modifyIndexConns.forEach(conn => { - assert.commandWorked( - conn.getCollection("config.transactions").dropIndex("parent_lsid")); - }); - } - rst.awaitReplication(); } assert.commandWorked(setFCVConn.getDB("foo").runCommand( @@ -111,15 +93,11 @@ function runTest( }); if (rst) { - // On step up to primary the index should not be created. Note this tests the non-empty - // collection case when alwaysCreateFeatureFlagEnabled is true. + // On step up to primary the index should not be created. let primary = rst.getPrimary(); - // Clear the collection so we'd try to create the index. Skip if the always create index - // feature flag is on because we'd try to create the index anyway. - if (!alwaysCreateFeatureFlagEnabled) { - assert.commandWorked(primary.getDB("config").transactions.remove({})); - } + // Clear the collection so we'd try to create the index. + assert.commandWorked(primary.getDB("config").transactions.remove({})); assert.commandWorked( primary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true})); assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); @@ -129,21 +107,8 @@ function runTest( rst.awaitReplication(); verifyIndexConns.forEach(conn => { reconnect(conn); - if (alwaysCreateFeatureFlagEnabled) { - assertPartialIndexExists(conn); - } else { - assertPartialIndexDoesNotExist(conn); - } + assertPartialIndexDoesNotExist(conn); }); - - if (alwaysCreateFeatureFlagEnabled) { - // The test expects no index after this block, so remove it. - modifyIndexConns.forEach(conn => { - assert.commandWorked( - conn.getCollection("config.transactions").dropIndex("parent_lsid")); - }); - } - rst.awaitReplication(); } assert.commandWorked(setFCVConn.getDB("foo").runCommand( @@ -159,11 +124,8 @@ function runTest( // On step up to primary the index should be created. let primary = rst.getPrimary(); - // Clear the collection so we'd try to create the index. Skip if the always create index - // feature flag is on because we'd try to create the index anyway. - if (!alwaysCreateFeatureFlagEnabled) { - assert.commandWorked(primary.getDB("config").transactions.remove({})); - } + // Clear the collection so we'll try to create the index. + assert.commandWorked(primary.getDB("config").transactions.remove({})); assert.commandWorked( primary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true})); assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); @@ -231,26 +193,6 @@ function runTest( } { - // Enabling featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp should not lead to - // creating the index if the internal transactions feature flag is not enabled. - const featureFlagRst = new ReplSetTest({ - nodes: 2, - nodeOptions: - {setParameter: "featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp=true"} - }); - featureFlagRst.startSet(); - featureFlagRst.initiate(); - // Note setFCV always waits for majority write concern so in a two node cluster secondaries will - // always have replicated the setFCV writes. - runTest(featureFlagRst.getPrimary(), - [featureFlagRst.getPrimary()], - [featureFlagRst.getPrimary(), featureFlagRst.getSecondary()], - featureFlagRst, - true /* alwaysCreateFeatureFlagEnabled */); - featureFlagRst.stopSet(); -} - -{ const conn = MongoRunner.runMongod(); const configTxnsCollection = conn.getCollection("config.transactions"); diff --git a/jstests/multiVersion/targetedTestsLastLtsFeatures/internal_transactions_transient_error_after_setFCV.js b/jstests/multiVersion/targetedTestsLastLtsFeatures/internal_transactions_transient_error_after_setFCV.js index 344c4df64b4..eccf64190d6 100644 --- a/jstests/multiVersion/targetedTestsLastLtsFeatures/internal_transactions_transient_error_after_setFCV.js +++ b/jstests/multiVersion/targetedTestsLastLtsFeatures/internal_transactions_transient_error_after_setFCV.js @@ -31,8 +31,6 @@ assert.commandWorked(st.s.adminCommand({split: ns, middle: {_id: 0}})); assert.commandWorked( st.s.adminCommand({moveChunk: ns, find: {_id: 0}, to: participant1.shardName})); -st.refreshCatalogCacheForNs(st.s, ns); - function runTest(lsid) { jsTest.log("Test that the correct error response is propagated upon losing in memory " + "transaction metadata and durable metadata in the config.transactions collection " + @@ -41,6 +39,9 @@ function runTest(lsid) { // Upgrade fcv to make sure cluster is on the latestFCV before starting any transactions. assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: latestFCV})); + // Upgrading to lastestFCV clears the filtering metadata of all collections. + st.refreshCatalogCacheForNs(st.s, ns); + // Inserts are split to guarantee that shard0 will be chosen as the coordinator. assert.commandWorked(st.s.getDB(kDbName).runCommand({ insert: kCollName, diff --git a/jstests/sharding/internal_txns/partial_index.js b/jstests/sharding/internal_txns/partial_index.js index b9c5462aaa4..c6fbb997b85 100644 --- a/jstests/sharding/internal_txns/partial_index.js +++ b/jstests/sharding/internal_txns/partial_index.js @@ -9,327 +9,285 @@ load("jstests/libs/analyze_plan.js"); +const st = new ShardingTest({shards: {rs0: {nodes: 2}}}); + const kDbName = "testDb"; const kCollName = "testColl"; const kConfigTxnNs = "config.transactions"; const kPartialIndexName = "parent_lsid"; -function runTest(st, alwaysCreateFeatureFlagEnabled) { - const mongosTestDB = st.s.getDB(kDbName); - const shard0PrimaryConfigTxnColl = st.rs0.getPrimary().getCollection(kConfigTxnNs); - - function assertPartialIndexExists(node) { - const configDB = node.getDB("config"); - const indexSpecs = - assert.commandWorked(configDB.runCommand({"listIndexes": "transactions"})) - .cursor.firstBatch; - indexSpecs.sort((index0, index1) => index0.name > index1.name); - assert.eq(indexSpecs.length, 2); - const idIndexSpec = indexSpecs[0]; - assert.eq(idIndexSpec.key, {"_id": 1}); - const partialIndexSpec = indexSpecs[1]; - assert.eq(partialIndexSpec.key, {"parentLsid": 1, "_id.txnNumber": 1, "_id": 1}); - assert.eq(partialIndexSpec.partialFilterExpression, {"parentLsid": {"$exists": true}}); - } - - function assertFindUsesCoveredQuery(node) { - const configTxnColl = node.getCollection(kConfigTxnNs); - const childSessionDoc = configTxnColl.findOne({ - "_id.id": sessionUUID, - "_id.txnNumber": childLsid.txnNumber, - "_id.txnUUID": childLsid.txnUUID - }); - - const explainRes = assert.commandWorked( - configTxnColl.explain() - .find({"parentLsid": parentSessionDoc._id, "_id.txnNumber": childLsid.txnNumber}, - {_id: 1}) - .finish()); - const winningPlan = getWinningPlan(explainRes.queryPlanner); - assert.eq(winningPlan.stage, "PROJECTION_COVERED"); - assert.eq(winningPlan.inputStage.stage, "IXSCAN"); - - const findRes = - configTxnColl - .find({"parentLsid": parentSessionDoc._id, "_id.txnNumber": childLsid.txnNumber}, - {_id: 1}) - .toArray(); - assert.eq(findRes.length, 1); - assert.eq(findRes[0]._id, childSessionDoc._id); - } - - function assertPartialIndexDoesNotExist(node) { - const configDB = node.getDB("config"); - const indexSpecs = - assert.commandWorked(configDB.runCommand({"listIndexes": "transactions"})) - .cursor.firstBatch; - assert.eq(indexSpecs.length, 1); - const idIndexSpec = indexSpecs[0]; - assert.eq(idIndexSpec.key, {"_id": 1}); - } - - function indexRecreationTest(expectRecreateAfterDrop) { - st.rs0.getPrimary().getCollection(kConfigTxnNs).dropIndex(kPartialIndexName); - st.rs0.awaitReplication(); - - st.rs0.nodes.forEach(node => { - assertPartialIndexDoesNotExist(node); - }); - - let primary = st.rs0.getPrimary(); - assert.commandWorked( - primary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true})); - assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); - - st.rs0.awaitNodesAgreeOnPrimary(); - st.rs0.awaitReplication(); - - st.rs0.nodes.forEach(node => { - if (expectRecreateAfterDrop) { - assertPartialIndexExists(node); - } else { - assertPartialIndexDoesNotExist(node); - } - }); - } - - // If the collection is empty and the index does not exist, we should always create the partial - // index on stepup, - indexRecreationTest(true /* expectRecreateAfterDrop */); - - const sessionUUID = UUID(); - const parentLsid = {id: sessionUUID}; - const parentTxnNumber = 35; - let stmtId = 0; - - assert.commandWorked(mongosTestDB.runCommand({ - insert: kCollName, - documents: [{_id: 0}], - lsid: parentLsid, - txnNumber: NumberLong(parentTxnNumber), - stmtId: NumberInt(stmtId++) - })); - const parentSessionDoc = shard0PrimaryConfigTxnColl.findOne({"_id.id": sessionUUID}); - - const childLsid = {id: sessionUUID, txnNumber: NumberLong(parentTxnNumber), txnUUID: UUID()}; - let childTxnNumber = 0; - - function runRetryableInternalTransaction(txnNumber) { - assert.commandWorked(mongosTestDB.runCommand({ - insert: kCollName, - documents: [{x: 1}], - lsid: childLsid, - txnNumber: NumberLong(txnNumber), - stmtId: NumberInt(stmtId++), - autocommit: false, - startTransaction: true - })); - assert.commandWorked(mongosTestDB.adminCommand({ - commitTransaction: 1, - lsid: childLsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - })); - } - - runRetryableInternalTransaction(childTxnNumber); - assert.eq(shard0PrimaryConfigTxnColl.count({"_id.id": sessionUUID}), 2); +const mongosTestDB = st.s.getDB(kDbName); +const shard0PrimaryConfigTxnColl = st.rs0.getPrimary().getCollection(kConfigTxnNs); + +function assertPartialIndexExists(node) { + const configDB = node.getDB("config"); + const indexSpecs = assert.commandWorked(configDB.runCommand({"listIndexes": "transactions"})) + .cursor.firstBatch; + indexSpecs.sort((index0, index1) => index0.name > index1.name); + assert.eq(indexSpecs.length, 2); + const idIndexSpec = indexSpecs[0]; + assert.eq(idIndexSpec.key, {"_id": 1}); + const partialIndexSpec = indexSpecs[1]; + assert.eq(partialIndexSpec.key, {"parentLsid": 1, "_id.txnNumber": 1, "_id": 1}); + assert.eq(partialIndexSpec.partialFilterExpression, {"parentLsid": {"$exists": true}}); +} - st.rs0.nodes.forEach(node => { - assertPartialIndexExists(node); - assertFindUsesCoveredQuery(node); +function assertFindUsesCoveredQuery(node) { + const configTxnColl = node.getCollection(kConfigTxnNs); + const childSessionDoc = configTxnColl.findOne({ + "_id.id": sessionUUID, + "_id.txnNumber": childLsid.txnNumber, + "_id.txnUUID": childLsid.txnUUID }); - childTxnNumber++; - runRetryableInternalTransaction(childTxnNumber); - assert.eq(shard0PrimaryConfigTxnColl.count({"_id.id": sessionUUID}), 2); + const explainRes = assert.commandWorked( + configTxnColl.explain() + .find({"parentLsid": parentSessionDoc._id, "_id.txnNumber": childLsid.txnNumber}, + {_id: 1}) + .finish()); + const winningPlan = getWinningPlan(explainRes.queryPlanner); + assert.eq(winningPlan.stage, "PROJECTION_COVERED"); + assert.eq(winningPlan.inputStage.stage, "IXSCAN"); + + const findRes = + configTxnColl + .find({"parentLsid": parentSessionDoc._id, "_id.txnNumber": childLsid.txnNumber}, + {_id: 1}) + .toArray(); + assert.eq(findRes.length, 1); + assert.eq(findRes[0]._id, childSessionDoc._id); +} + +function assertPartialIndexDoesNotExist(node) { + const configDB = node.getDB("config"); + const indexSpecs = assert.commandWorked(configDB.runCommand({"listIndexes": "transactions"})) + .cursor.firstBatch; + assert.eq(indexSpecs.length, 1); + const idIndexSpec = indexSpecs[0]; + assert.eq(idIndexSpec.key, {"_id": 1}); +} + +function indexRecreationTest(recreateAfterDrop) { + st.rs0.getPrimary().getCollection(kConfigTxnNs).dropIndex(kPartialIndexName); + st.rs0.awaitReplication(); st.rs0.nodes.forEach(node => { - assertPartialIndexExists(node); - assertFindUsesCoveredQuery(node); + assertPartialIndexDoesNotExist(node); }); - // - // Verify clients can create the index only if they provide the exact specification and that - // operations requiring the index fails if it does not exist. - // - - const indexConn = st.rs0.getPrimary(); + let primary = st.rs0.getPrimary(); assert.commandWorked( - indexConn.getCollection("config.transactions").dropIndex(kPartialIndexName)); - - // Normal writes don't involve config.transactions, so they succeed. - assert.commandWorked(indexConn.getDB(kDbName).runCommand( - {insert: kCollName, documents: [{x: 1}], lsid: {id: UUID()}})); - - // Retryable writes read from the partial index, so they fail. - let res = assert.commandFailedWithCode(indexConn.getDB(kDbName).runCommand({ - insert: kCollName, - documents: [{x: 1}], - lsid: {id: UUID()}, - txnNumber: NumberLong(11) - }), - ErrorCodes.BadValue); - assert(res.errmsg.includes("Please create an index directly "), tojson(res)); - - // User transactions read from the partial index, so they fail. - assert.commandFailedWithCode(indexConn.getDB(kDbName).runCommand({ - insert: kCollName, - documents: [{x: 1}], - lsid: {id: UUID()}, - txnNumber: NumberLong(11), - startTransaction: true, - autocommit: false - }), - ErrorCodes.BadValue); - - // Non retryable internal transactions do not read from or update the partial index, so they can - // succeed without the index existing. - let nonRetryableTxnSession = {id: UUID(), txnUUID: UUID()}; - assert.commandWorked(indexConn.getDB(kDbName).runCommand({ - insert: kCollName, - documents: [{x: 1}], - lsid: nonRetryableTxnSession, - txnNumber: NumberLong(11), - stmtId: NumberInt(0), - startTransaction: true, - autocommit: false - })); - assert.commandWorked(indexConn.adminCommand({ - commitTransaction: 1, - lsid: nonRetryableTxnSession, - txnNumber: NumberLong(11), - autocommit: false - })); - - // Retryable transactions read from the partial index, so they fail. - assert.commandFailedWithCode(indexConn.getDB(kDbName).runCommand({ - insert: kCollName, - documents: [{x: 1}], - lsid: {id: UUID(), txnUUID: UUID(), txnNumber: NumberLong(2)}, - txnNumber: NumberLong(11), - stmtId: NumberInt(0), - startTransaction: true, - autocommit: false - }), - ErrorCodes.BadValue); - - // Recreating the partial index requires the exact options used internally, but in any order. - assert.commandFailedWithCode(indexConn.getDB("config").runCommand({ - createIndexes: "transactions", - indexes: [{v: 2, name: "parent_lsid", key: {parentLsid: 1, "_id.txnNumber": 1, _id: 1}}], - }), - ErrorCodes.IllegalOperation); - assert.commandWorked(indexConn.getDB("config").runCommand({ - createIndexes: "transactions", - indexes: [{ - name: "parent_lsid", - key: {parentLsid: 1, "_id.txnNumber": 1, _id: 1}, - partialFilterExpression: {parentLsid: {$exists: true}}, - v: 2, - }], - })); - - // Operations involving the index should succeed now. - - assert.commandWorked(indexConn.getDB(kDbName).runCommand( - {insert: kCollName, documents: [{x: 1}], lsid: {id: UUID()}})); - - assert.commandWorked(indexConn.getDB(kDbName).runCommand( - {insert: kCollName, documents: [{x: 1}], lsid: {id: UUID()}, txnNumber: NumberLong(11)})); + primary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true})); + assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); - let userSessionAfter = {id: UUID()}; - assert.commandWorked(indexConn.getDB(kDbName).runCommand({ - insert: kCollName, - documents: [{x: 1}], - lsid: userSessionAfter, - txnNumber: NumberLong(11), - startTransaction: true, - autocommit: false - })); - assert.commandWorked(indexConn.adminCommand({ - commitTransaction: 1, - lsid: userSessionAfter, - txnNumber: NumberLong(11), - autocommit: false - })); + st.rs0.awaitNodesAgreeOnPrimary(); + st.rs0.awaitReplication(); - let nonRetryableTxnSessionAfter = {id: UUID(), txnUUID: UUID()}; - assert.commandWorked(indexConn.getDB(kDbName).runCommand({ - insert: kCollName, - documents: [{x: 1}], - lsid: nonRetryableTxnSessionAfter, - txnNumber: NumberLong(11), - stmtId: NumberInt(0), - startTransaction: true, - autocommit: false - })); - assert.commandWorked(indexConn.adminCommand({ - commitTransaction: 1, - lsid: nonRetryableTxnSessionAfter, - txnNumber: NumberLong(11), - autocommit: false - })); + st.rs0.nodes.forEach(node => { + if (recreateAfterDrop) { + assertPartialIndexExists(node); + } else { + assertPartialIndexDoesNotExist(node); + } + }); +} - let retryableTxnSessionAfter = {id: UUID(), txnUUID: UUID(), txnNumber: NumberLong(2)}; - assert.commandWorked(indexConn.getDB(kDbName).runCommand({ +// If the collection is empty and the index does not exist, we should create the partial index on +// stepup. +indexRecreationTest(true /*Recreate after drop*/); + +const sessionUUID = UUID(); +const parentLsid = { + id: sessionUUID +}; +const parentTxnNumber = 35; +let stmtId = 0; + +assert.commandWorked(mongosTestDB.runCommand({ + insert: kCollName, + documents: [{_id: 0}], + lsid: parentLsid, + txnNumber: NumberLong(parentTxnNumber), + stmtId: NumberInt(stmtId++) +})); +const parentSessionDoc = shard0PrimaryConfigTxnColl.findOne({"_id.id": sessionUUID}); + +const childLsid = { + id: sessionUUID, + txnNumber: NumberLong(parentTxnNumber), + txnUUID: UUID() +}; +let childTxnNumber = 0; + +function runRetryableInternalTransaction(txnNumber) { + assert.commandWorked(mongosTestDB.runCommand({ insert: kCollName, documents: [{x: 1}], - lsid: retryableTxnSessionAfter, - txnNumber: NumberLong(11), - stmtId: NumberInt(0), - startTransaction: true, - autocommit: false + lsid: childLsid, + txnNumber: NumberLong(txnNumber), + stmtId: NumberInt(stmtId++), + autocommit: false, + startTransaction: true })); - assert.commandWorked(indexConn.adminCommand({ + assert.commandWorked(mongosTestDB.adminCommand({ commitTransaction: 1, - lsid: retryableTxnSessionAfter, - txnNumber: NumberLong(11), + lsid: childLsid, + txnNumber: NumberLong(txnNumber), autocommit: false })); - - if (!alwaysCreateFeatureFlagEnabled) { - // We expect that if the partial index is dropped when the collection isn't empty, then on - // stepup we should not recreate the collection. - indexRecreationTest(false /* expectRecreateAfterDrop */); - } else { - // Creating the partial index when the collection isn't empty can be enabled by a feature - // flag. - indexRecreationTest(true /* expectRecreateAfterDrop */); - } -} - -{ - const st = new ShardingTest({shards: {rs0: {nodes: 2}}}); - runTest(st, false /* alwaysCreateFeatureFlagEnabled */); - st.stop(); } -{ - const featureFlagSt = new ShardingTest({ - shards: 1, - other: { - rs: {nodes: 2}, - rsOptions: - {setParameter: "featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp=true"} - } - }); - - // Sanity check the feature flag was enabled. - assert(assert - .commandWorked(featureFlagSt.rs0.getPrimary().adminCommand({ - getParameter: 1, - featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp: 1 - })) - .featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp.value); - assert(assert - .commandWorked(featureFlagSt.rs0.getSecondary().adminCommand({ - getParameter: 1, - featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp: 1 - })) - .featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp.value); - - runTest(featureFlagSt, true /* alwaysCreateFeatureFlagEnabled */); - featureFlagSt.stop(); -} +runRetryableInternalTransaction(childTxnNumber); +assert.eq(shard0PrimaryConfigTxnColl.count({"_id.id": sessionUUID}), 2); + +st.rs0.nodes.forEach(node => { + assertPartialIndexExists(node); + assertFindUsesCoveredQuery(node); +}); + +childTxnNumber++; +runRetryableInternalTransaction(childTxnNumber); +assert.eq(shard0PrimaryConfigTxnColl.count({"_id.id": sessionUUID}), 2); + +st.rs0.nodes.forEach(node => { + assertPartialIndexExists(node); + assertFindUsesCoveredQuery(node); +}); + +// +// Verify clients can create the index only if they provide the exact specification and that +// operations requiring the index fails if it does not exist. +// + +const indexConn = st.rs0.getPrimary(); +assert.commandWorked(indexConn.getCollection("config.transactions").dropIndex(kPartialIndexName)); + +// Normal writes don't involve config.transactions, so they succeed. +assert.commandWorked(indexConn.getDB(kDbName).runCommand( + {insert: kCollName, documents: [{x: 1}], lsid: {id: UUID()}})); + +// Retryable writes read from the partial index, so they fail. +let res = assert.commandFailedWithCode( + indexConn.getDB(kDbName).runCommand( + {insert: kCollName, documents: [{x: 1}], lsid: {id: UUID()}, txnNumber: NumberLong(11)}), + ErrorCodes.BadValue); +assert(res.errmsg.includes("Please create an index directly "), tojson(res)); + +// User transactions read from the partial index, so they fail. +assert.commandFailedWithCode(indexConn.getDB(kDbName).runCommand({ + insert: kCollName, + documents: [{x: 1}], + lsid: {id: UUID()}, + txnNumber: NumberLong(11), + startTransaction: true, + autocommit: false +}), + ErrorCodes.BadValue); + +// Non retryable internal transactions do not read from or update the partial index, so they can +// succeed without the index existing. +let nonRetryableTxnSession = {id: UUID(), txnUUID: UUID()}; +assert.commandWorked(indexConn.getDB(kDbName).runCommand({ + insert: kCollName, + documents: [{x: 1}], + lsid: nonRetryableTxnSession, + txnNumber: NumberLong(11), + stmtId: NumberInt(0), + startTransaction: true, + autocommit: false +})); +assert.commandWorked(indexConn.adminCommand({ + commitTransaction: 1, + lsid: nonRetryableTxnSession, + txnNumber: NumberLong(11), + autocommit: false +})); + +// Retryable transactions read from the partial index, so they fail. +assert.commandFailedWithCode(indexConn.getDB(kDbName).runCommand({ + insert: kCollName, + documents: [{x: 1}], + lsid: {id: UUID(), txnUUID: UUID(), txnNumber: NumberLong(2)}, + txnNumber: NumberLong(11), + stmtId: NumberInt(0), + startTransaction: true, + autocommit: false +}), + ErrorCodes.BadValue); + +// Recreating the partial index requires the exact options used internally, but in any order. +assert.commandFailedWithCode(indexConn.getDB("config").runCommand({ + createIndexes: "transactions", + indexes: [{v: 2, name: "parent_lsid", key: {parentLsid: 1, "_id.txnNumber": 1, _id: 1}}], +}), + ErrorCodes.IllegalOperation); +assert.commandWorked(indexConn.getDB("config").runCommand({ + createIndexes: "transactions", + indexes: [{ + name: "parent_lsid", + key: {parentLsid: 1, "_id.txnNumber": 1, _id: 1}, + partialFilterExpression: {parentLsid: {$exists: true}}, + v: 2, + }], +})); + +// Operations involving the index should succeed now. + +assert.commandWorked(indexConn.getDB(kDbName).runCommand( + {insert: kCollName, documents: [{x: 1}], lsid: {id: UUID()}})); + +assert.commandWorked(indexConn.getDB(kDbName).runCommand( + {insert: kCollName, documents: [{x: 1}], lsid: {id: UUID()}, txnNumber: NumberLong(11)})); + +let userSessionAfter = {id: UUID()}; +assert.commandWorked(indexConn.getDB(kDbName).runCommand({ + insert: kCollName, + documents: [{x: 1}], + lsid: userSessionAfter, + txnNumber: NumberLong(11), + startTransaction: true, + autocommit: false +})); +assert.commandWorked(indexConn.adminCommand( + {commitTransaction: 1, lsid: userSessionAfter, txnNumber: NumberLong(11), autocommit: false})); + +let nonRetryableTxnSessionAfter = {id: UUID(), txnUUID: UUID()}; +assert.commandWorked(indexConn.getDB(kDbName).runCommand({ + insert: kCollName, + documents: [{x: 1}], + lsid: nonRetryableTxnSessionAfter, + txnNumber: NumberLong(11), + stmtId: NumberInt(0), + startTransaction: true, + autocommit: false +})); +assert.commandWorked(indexConn.adminCommand({ + commitTransaction: 1, + lsid: nonRetryableTxnSessionAfter, + txnNumber: NumberLong(11), + autocommit: false +})); + +let retryableTxnSessionAfter = {id: UUID(), txnUUID: UUID(), txnNumber: NumberLong(2)}; +assert.commandWorked(indexConn.getDB(kDbName).runCommand({ + insert: kCollName, + documents: [{x: 1}], + lsid: retryableTxnSessionAfter, + txnNumber: NumberLong(11), + stmtId: NumberInt(0), + startTransaction: true, + autocommit: false +})); +assert.commandWorked(indexConn.adminCommand({ + commitTransaction: 1, + lsid: retryableTxnSessionAfter, + txnNumber: NumberLong(11), + autocommit: false +})); + +// We expect that if the partial index is dropped when the collection isn't empty, then on stepup we +// should not recreate the collection. +indexRecreationTest(false /*Don't recreate after drop*/); + +st.stop(); })(); diff --git a/src/mongo/base/error_codes.yml b/src/mongo/base/error_codes.yml index 0b3150042bd..523c52065cf 100644 --- a/src/mongo/base/error_codes.yml +++ b/src/mongo/base/error_codes.yml @@ -490,7 +490,6 @@ error_codes: - {code: 374, name: TransactionAPIMustRetryTransaction, categories: [InternalOnly]} - {code: 375, name: TransactionAPIMustRetryCommit, categories: [InternalOnly]} - - {code: 377, name: FLEMaxTagLimitExceeded } # Error codes 4000-8999 are reserved. diff --git a/src/mongo/crypto/encryption_fields.idl b/src/mongo/crypto/encryption_fields.idl index f8205b1f156..903a1d4f415 100644 --- a/src/mongo/crypto/encryption_fields.idl +++ b/src/mongo/crypto/encryption_fields.idl @@ -58,7 +58,7 @@ structs: contention: description: "Contention factor for field, 0 means it has extremely high set number of distinct values" type: exactInt64 - default: 4 + default: 0 unstable: true validator: { gte: 0 } diff --git a/src/mongo/crypto/fle_crypto.cpp b/src/mongo/crypto/fle_crypto.cpp index f55db25f970..38800792351 100644 --- a/src/mongo/crypto/fle_crypto.cpp +++ b/src/mongo/crypto/fle_crypto.cpp @@ -53,7 +53,6 @@ #include "mongo/base/error_codes.h" #include "mongo/base/status.h" #include "mongo/bson/bson_depth.h" -#include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/bsontypes.h" @@ -154,8 +153,6 @@ PrfBlock blockToArray(const SHA256Block& block) { return data; } -} // namespace - PrfBlock PrfBlockfromCDR(ConstDataRange block) { uassert(6373501, "Invalid prf length", block.length() == sizeof(PrfBlock)); @@ -164,7 +161,6 @@ PrfBlock PrfBlockfromCDR(ConstDataRange block) { return ret; } -namespace { ConstDataRange hmacKey(const KeyMaterial& keyMaterial) { static_assert(kHmacKeyOffset + crypto::sym256KeySize <= crypto::kFieldLevelEncryptionKeySize); invariant(crypto::kFieldLevelEncryptionKeySize == keyMaterial->size()); @@ -216,16 +212,13 @@ ConstDataRange binDataToCDR(const BSONElement element) { return ConstDataRange(data, data + len); } -ConstDataRange binDataToCDR(const BSONBinData binData) { - int len = binData.length; - const char* data = static_cast<const char*>(binData.data); - return ConstDataRange(data, data + len); -} - ConstDataRange binDataToCDR(const Value& value) { uassert(6334103, "Expected binData Value type", value.getType() == BinData); - return binDataToCDR(value.getBinData()); + auto binData = value.getBinData(); + int len = binData.length; + const char* data = static_cast<const char*>(binData.data); + return ConstDataRange(data, data + len); } template <typename T> @@ -299,7 +292,7 @@ void toEncryptedBinData(StringData field, std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedBinData(BSONElement element) { uassert( - 6672414, "Expected binData with subtype Encrypt", element.isBinData(BinDataType::Encrypt)); + 6373502, "Expected binData with subtype Encrypt", element.isBinData(BinDataType::Encrypt)); return fromEncryptedConstDataRange(binDataToCDR(element)); } @@ -972,6 +965,7 @@ void parseAndVerifyInsertUpdatePayload(std::vector<EDCServerPayloadInfo>* pField void collectEDCServerInfo(std::vector<EDCServerPayloadInfo>* pFields, ConstDataRange cdr, + StringData fieldPath) { // TODO - validate field is actually indexed in the schema? @@ -1169,28 +1163,6 @@ uint64_t generateRandomContention(uint64_t cm) { } // namespace -std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedBinData(const Value& value) { - uassert(6672416, "Expected binData with subtype Encrypt", value.getType() == BinData); - - auto binData = value.getBinData(); - - uassert(6672415, "Expected binData with subtype Encrypt", binData.type == BinDataType::Encrypt); - - return fromEncryptedConstDataRange(binDataToCDR(binData)); -} - -BSONBinData toBSONBinData(const std::vector<uint8_t>& buf) { - return BSONBinData(buf.data(), buf.size(), Encrypt); -} - -std::vector<uint8_t> toEncryptedVector(EncryptedBinDataType dt, const PrfBlock& block) { - std::vector<uint8_t> buf(block.size() + 1); - buf[0] = static_cast<uint8_t>(dt); - - std::copy(block.data(), block.data() + block.size(), buf.data() + 1); - - return buf; -} CollectionsLevel1Token FLELevel1TokenGenerator::generateCollectionsLevel1Token( FLEIndexKey indexKey) { @@ -1392,8 +1364,6 @@ std::pair<BSONType, std::vector<uint8_t>> FLEClientCrypto::decrypt(ConstDataRang return {EOO, vectorFromCDR(pair.second)}; } else if (pair.first == EncryptedBinDataType::kFLE2InsertUpdatePayload) { return {EOO, vectorFromCDR(pair.second)}; - } else if (pair.first == EncryptedBinDataType::kFLE2TransientRaw) { - return {EOO, vectorFromCDR(pair.second)}; } else { uasserted(6373507, "Not supported"); } @@ -1750,8 +1720,6 @@ FLE2FindEqualityPayload FLEClientCrypto::serializeFindPayload(FLEIndexKeyAndId i auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(indexKey.key); - auto serverToken = - FLELevel1TokenGenerator::generateServerDataEncryptionLevel1Token(indexKey.key); auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); auto escToken = FLECollectionTokenGenerator::generateESCToken(collectionToken); @@ -1770,7 +1738,6 @@ FLE2FindEqualityPayload FLEClientCrypto::serializeFindPayload(FLEIndexKeyAndId i payload.setEscDerivedToken(escDatakey.toCDR()); payload.setEccDerivedToken(eccDatakey.toCDR()); payload.setMaxCounter(maxContentionFactor); - payload.setServerEncryptionToken(serverToken.toCDR()); return payload; } @@ -2052,8 +2019,7 @@ ESCDerivedFromDataTokenAndContentionFactorToken EDCServerPayloadInfo::getESCToke } void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, - const EncryptedFieldConfig& efc, - bool bypassDocumentValidation) { + const EncryptedFieldConfig& efc) { stdx::unordered_set<std::string> indexedFields; for (auto f : efc.getFields()) { if (f.getQueries().has_value()) { @@ -2070,11 +2036,6 @@ void EDCServerCollection::validateEncryptedFieldInfo(BSONObj& obj, indexedFields.contains(fieldPath.toString())); } }); - - // We should ensure that the user is not manually modifying the safe content array. - uassert(6666200, - str::stream() << "Cannot modify " << kSafeContent << " field in document.", - !obj.hasField(kSafeContent) || bypassDocumentValidation); } @@ -2115,44 +2076,6 @@ PrfBlock EDCServerCollection::generateTag(const FLE2IndexedEqualityEncryptedValu return generateTag(edcTwiceDerived, indexedValue.count); } - -StatusWith<FLE2IndexedEqualityEncryptedValue> EDCServerCollection::decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue) { - auto pair = fromEncryptedConstDataRange(serializedServerValue); - uassert(6672412, - "Wrong encrypted field type", - pair.first == EncryptedBinDataType::kFLE2EqualityIndexedValue); - - return FLE2IndexedEqualityEncryptedValue::decryptAndParse(token, pair.second); -} - -StatusWith<FLE2IndexedEqualityEncryptedValue> EDCServerCollection::decryptAndParse( - ConstDataRange token, ConstDataRange serializedServerValue) { - auto serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>(token); - - return FLE2IndexedEqualityEncryptedValue::decryptAndParse(serverToken, serializedServerValue); -} - -std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> EDCServerCollection::generateEDCTokens( - EDCDerivedFromDataToken token, uint64_t maxContentionFactor) { - std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> tokens; - tokens.reserve(maxContentionFactor); - - for (uint64_t i = 0; i <= maxContentionFactor; ++i) { - tokens.push_back(FLEDerivedFromDataTokenAndContentionFactorTokenGenerator:: - generateEDCDerivedFromDataTokenAndContentionFactorToken(token, i)); - } - - return tokens; -} - -std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> EDCServerCollection::generateEDCTokens( - ConstDataRange rawToken, uint64_t maxContentionFactor) { - auto token = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(rawToken); - - return generateEDCTokens(token, maxContentionFactor); -} - BSONObj EDCServerCollection::finalizeForInsert( const BSONObj& doc, const std::vector<EDCServerPayloadInfo>& serverPayload) { std::vector<TagInfo> tags; @@ -2382,7 +2305,6 @@ EncryptedFieldConfig EncryptionInformationHelpers::getAndValidateSchema( return efc; } - std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(ConstDataRange cdr) { ConstDataRangeCursor cdrc(cdr); @@ -2455,12 +2377,6 @@ ParsedFindPayload::ParsedFindPayload(ConstDataRange cdr) { escToken = FLETokenFromCDR<FLETokenType::ESCDerivedFromDataToken>(payload.getEscDerivedToken()); eccToken = FLETokenFromCDR<FLETokenType::ECCDerivedFromDataToken>(payload.getEccDerivedToken()); edcToken = FLETokenFromCDR<FLETokenType::EDCDerivedFromDataToken>(payload.getEdcDerivedToken()); - - if (payload.getServerEncryptionToken().has_value()) { - serverToken = FLETokenFromCDR<FLETokenType::ServerDataEncryptionLevel1Token>( - payload.getServerEncryptionToken().value()); - } - maxCounter = payload.getMaxCounter(); } diff --git a/src/mongo/crypto/fle_crypto.h b/src/mongo/crypto/fle_crypto.h index 5feac8ca2d3..5d8285f5790 100644 --- a/src/mongo/crypto/fle_crypto.h +++ b/src/mongo/crypto/fle_crypto.h @@ -41,7 +41,6 @@ #include "mongo/base/status_with.h" #include "mongo/base/string_data.h" #include "mongo/bson/bsonelement.h" -#include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsontypes.h" #include "mongo/crypto/aead_encryption.h" @@ -1010,21 +1009,13 @@ public: /** * Validate that payload is compatible with schema */ - static void validateEncryptedFieldInfo(BSONObj& obj, - const EncryptedFieldConfig& efc, - bool bypassDocumentValidation); + static void validateEncryptedFieldInfo(BSONObj& obj, const EncryptedFieldConfig& efc); /** * Get information about all FLE2InsertUpdatePayload payloads */ static std::vector<EDCServerPayloadInfo> getEncryptedFieldInfo(BSONObj& obj); - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ServerDataEncryptionLevel1Token token, ConstDataRange serializedServerValue); - - static StatusWith<FLE2IndexedEqualityEncryptedValue> decryptAndParse( - ConstDataRange token, ConstDataRange serializedServerValue); - /** * Generate a search tag * @@ -1035,14 +1026,6 @@ public: static PrfBlock generateTag(const FLE2IndexedEqualityEncryptedValue& indexedValue); /** - * Generate all the EDC tokens - */ - static std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> generateEDCTokens( - EDCDerivedFromDataToken token, uint64_t maxContentionFactor); - static std::vector<EDCDerivedFromDataTokenAndContentionFactorToken> generateEDCTokens( - ConstDataRange rawToken, uint64_t maxContentionFactor); - - /** * Consumes a payload from a MongoDB client for insert. * * Converts FLE2InsertUpdatePayload to a final insert payload and updates __safeContent__ with @@ -1180,7 +1163,6 @@ struct ParsedFindPayload { ESCDerivedFromDataToken escToken; ECCDerivedFromDataToken eccToken; EDCDerivedFromDataToken edcToken; - boost::optional<ServerDataEncryptionLevel1Token> serverToken; boost::optional<std::int64_t> maxCounter; explicit ParsedFindPayload(BSONElement fleFindPayload); @@ -1188,15 +1170,4 @@ struct ParsedFindPayload { explicit ParsedFindPayload(ConstDataRange cdr); }; -/** - * Utility functions manipulating buffers - */ -PrfBlock PrfBlockfromCDR(ConstDataRange block); - -std::vector<uint8_t> toEncryptedVector(EncryptedBinDataType dt, const PrfBlock& block); - -BSONBinData toBSONBinData(const std::vector<uint8_t>& buf); - -std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedBinData(const Value& value); - } // namespace mongo diff --git a/src/mongo/crypto/fle_crypto_test.cpp b/src/mongo/crypto/fle_crypto_test.cpp index 4c4355ebb9f..75c1976c097 100644 --- a/src/mongo/crypto/fle_crypto_test.cpp +++ b/src/mongo/crypto/fle_crypto_test.cpp @@ -33,7 +33,6 @@ #include "mongo/crypto/fle_crypto.h" #include <algorithm> -#include <cstdint> #include <iostream> #include <limits> #include <stack> @@ -697,8 +696,7 @@ std::vector<char> generatePlaceholder( BSONElement value, Operation operation, mongo::Fle2AlgorithmInt algorithm = mongo::Fle2AlgorithmInt::kEquality, - boost::optional<UUID> key = boost::none, - uint64_t contention = 0) { + boost::optional<UUID> key = boost::none) { FLE2EncryptionPlaceholder ep; if (operation == Operation::kFind) { @@ -711,7 +709,7 @@ std::vector<char> generatePlaceholder( ep.setUserKeyId(userKeyId); ep.setIndexKeyId(key.value_or(indexKeyId)); ep.setValue(value); - ep.setMaxContentionCounter(contention); + ep.setMaxContentionCounter(0); BSONObj obj = ep.toBSON(); @@ -728,7 +726,7 @@ BSONObj encryptDocument(BSONObj obj, auto result = FLEClientCrypto::transformPlaceholders(obj, keyVault); if (nullptr != efc) { - EDCServerCollection::validateEncryptedFieldInfo(result, *efc, false); + EDCServerCollection::validateEncryptedFieldInfo(result, *efc); } // Start Server Side @@ -834,41 +832,6 @@ void roundTripMultiencrypted(BSONObj doc1, assertPayload(finalDoc["encrypted2"], operation2); } -// Used to generate the test data for the ExpressionFLETest in expression_test.cpp -TEST(FLE_EDC, PrintTest) { - auto doc = BSON("value" << 1); - auto element = doc.firstElement(); - - TestKeyVault keyVault; - - auto inputDoc = BSON("plainText" - << "sample" - << "encrypted" << element); - - { - auto buf = generatePlaceholder(element, Operation::kInsert, Fle2AlgorithmInt::kEquality); - BSONObjBuilder builder; - builder.append("plainText", "sample"); - builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); - - auto finalDoc = encryptDocument(builder.obj(), &keyVault); - - std::cout << finalDoc.jsonString() << std::endl; - } - - { - auto buf = generatePlaceholder( - element, Operation::kInsert, Fle2AlgorithmInt::kEquality, boost::none, 50); - BSONObjBuilder builder; - builder.append("plainText", "sample"); - builder.appendBinData("encrypted", buf.size(), BinDataType::Encrypt, buf.data()); - - auto finalDoc = encryptDocument(builder.obj(), &keyVault); - - std::cout << finalDoc.jsonString() << std::endl; - } -} - TEST(FLE_EDC, Allowed_Types) { const std::vector<std::pair<BSONObj, BSONType>> universallyAllowedObjects{ {BSON("sample" @@ -1965,25 +1928,4 @@ TEST(CompactionHelpersTest, countDeletedTest) { ASSERT_EQ(CompactionHelpers::countDeleted(input), 20); } -TEST(EDCServerCollectionTest, GenerateEDCTokens) { - - auto doc = BSON("sample" << 123456); - auto element = doc.firstElement(); - - auto value = ConstDataRange(element.value(), element.value() + element.valuesize()); - - auto collectionToken = FLELevel1TokenGenerator::generateCollectionsLevel1Token(getIndexKey()); - auto edcToken = FLECollectionTokenGenerator::generateEDCToken(collectionToken); - - EDCDerivedFromDataToken edcDatakey = - FLEDerivedFromDataTokenGenerator::generateEDCDerivedFromDataToken(edcToken, value); - - - ASSERT_EQ(EDCServerCollection::generateEDCTokens(edcDatakey, 0).size(), 1); - ASSERT_EQ(EDCServerCollection::generateEDCTokens(edcDatakey, 1).size(), 2); - ASSERT_EQ(EDCServerCollection::generateEDCTokens(edcDatakey, 2).size(), 3); - ASSERT_EQ(EDCServerCollection::generateEDCTokens(edcDatakey, 3).size(), 4); -} - - } // namespace mongo diff --git a/src/mongo/crypto/fle_field_schema.idl b/src/mongo/crypto/fle_field_schema.idl index d9e2b54b890..030fa36ef3f 100644 --- a/src/mongo/crypto/fle_field_schema.idl +++ b/src/mongo/crypto/fle_field_schema.idl @@ -51,10 +51,6 @@ enums: kFLE2UnindexedEncryptedValue : 6 # see FLE2IndexedEqualityEncryptedValue kFLE2EqualityIndexedValue : 7 - # Transient encrypted data in query rewrites, not persisted - # same as BinDataGeneral but redacted - kFLE2TransientRaw : 8 - FleVersion: description: "The version / type of field-level encryption in use." type: int diff --git a/src/mongo/crypto/fle_tags.cpp b/src/mongo/crypto/fle_tags.cpp index 4737ff13144..a0de37b2f42 100644 --- a/src/mongo/crypto/fle_tags.cpp +++ b/src/mongo/crypto/fle_tags.cpp @@ -56,11 +56,8 @@ void verifyTagsWillFit(size_t tagCount, size_t memoryLimit) { constexpr size_t largestElementSize = arrayElementSize(std::numeric_limits<size_t>::digits10); constexpr size_t ridiculousNumberOfTags = std::numeric_limits<size_t>::max() / largestElementSize; - - uassert(ErrorCodes::FLEMaxTagLimitExceeded, - "Encrypted rewrite too many tags", - tagCount < ridiculousNumberOfTags); - uassert(ErrorCodes::FLEMaxTagLimitExceeded, + uassert(6653300, "Encrypted rewrite too many tags", tagCount < ridiculousNumberOfTags); + uassert(6401800, "Encrypted rewrite memory limit exceeded", sizeArrayElementsMemory(tagCount) <= memoryLimit); } diff --git a/src/mongo/db/active_index_builds.cpp b/src/mongo/db/active_index_builds.cpp index d6a7f5afabf..017ca49ecc8 100644 --- a/src/mongo/db/active_index_builds.cpp +++ b/src/mongo/db/active_index_builds.cpp @@ -28,8 +28,6 @@ */ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage -#include <fmt/format.h> - #include "mongo/db/active_index_builds.h" #include "mongo/db/catalog/index_builds_manager.h" #include "mongo/logv2/log.h" @@ -65,14 +63,10 @@ void ActiveIndexBuilds::waitForAllIndexBuildsToStopForShutdown(OperationContext* void ActiveIndexBuilds::assertNoIndexBuildInProgress() const { stdx::unique_lock<Latch> lk(_mutex); - if (!_allIndexBuilds.empty()) { - auto firstIndexBuild = _allIndexBuilds.cbegin()->second; - uasserted(ErrorCodes::BackgroundOperationInProgressForDatabase, - fmt::format("cannot perform operation: there are currently {} index builds " - "running. Found index build: {}", - _allIndexBuilds.size(), - firstIndexBuild->buildUUID.toString())); - } + uassert(ErrorCodes::BackgroundOperationInProgressForDatabase, + str::stream() << "cannot perform operation: there are currently " + << _allIndexBuilds.size() << " index builds running.", + _allIndexBuilds.size() == 0); } void ActiveIndexBuilds::waitUntilAnIndexBuildFinishes(OperationContext* opCtx) { diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 8af0ba8efc9..ca5f29ae55e 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -175,11 +175,13 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati } if (auto& cappedSize = cmr.getCappedSize()) { + static constexpr long long minCappedSize = 4096; auto swCappedSize = CollectionOptions::checkAndAdjustCappedSize(*cappedSize); if (!swCappedSize.isOK()) { return swCappedSize.getStatus(); } - parsed.cappedSize = swCappedSize.getValue(); + parsed.cappedSize = + (swCappedSize.getValue() < minCappedSize) ? minCappedSize : swCappedSize.getValue(); oplogEntryBuilder.append(CollMod::kCappedSizeFieldName, *cappedSize); } if (auto& cappedMax = cmr.getCappedMax()) { diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index b79c8c78914..29e343e44bc 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -38,7 +38,6 @@ #include "mongo/bson/ordering.h" #include "mongo/bson/simple_bsonelement_comparator.h" #include "mongo/bson/simple_bsonobj_comparator.h" -#include "mongo/crypto/fle_crypto.h" #include "mongo/db/auth/security_token.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_options.h" @@ -817,6 +816,7 @@ Status CollectionImpl::insertDocumentsForOplog(OperationContext* opCtx, return status; } + Status CollectionImpl::insertDocuments(OperationContext* opCtx, const std::vector<InsertStatement>::const_iterator begin, const std::vector<InsertStatement>::const_iterator end, @@ -840,20 +840,8 @@ Status CollectionImpl::insertDocuments(OperationContext* opCtx, } auto status = _checkValidationAndParseResult(opCtx, it->doc); - if (!status.isOK()) { + if (!status.isOK()) return status; - } - - auto& validationSettings = DocumentValidationSettings::get(opCtx); - - if (getCollectionOptions().encryptedFieldConfig && - !validationSettings.isSchemaValidationDisabled() && - !validationSettings.isSafeContentValidationDisabled() && - it->doc.hasField(kSafeContent)) { - return Status(ErrorCodes::BadValue, - str::stream() - << "Cannot insert a document with field name " << kSafeContent); - } } const SnapshotId sid = opCtx->recoveryUnit()->getSnapshotId(); @@ -1354,17 +1342,6 @@ void CollectionImpl::deleteDocument(OperationContext* opCtx, } } -bool compareSafeContentElem(const BSONObj& oldDoc, const BSONObj& newDoc) { - if (newDoc.hasField(kSafeContent) != oldDoc.hasField(kSafeContent)) { - return false; - } - if (!newDoc.hasField(kSafeContent)) { - return true; - } - - return newDoc.getField(kSafeContent).binaryEqual(oldDoc.getField(kSafeContent)); -} - RecordId CollectionImpl::updateDocument(OperationContext* opCtx, RecordId oldLocation, const Snapshotted<BSONObj>& oldDoc, @@ -1389,17 +1366,6 @@ RecordId CollectionImpl::updateDocument(OperationContext* opCtx, } } - auto& validationSettings = DocumentValidationSettings::get(opCtx); - if (getCollectionOptions().encryptedFieldConfig && - !validationSettings.isSchemaValidationDisabled() && - !validationSettings.isSafeContentValidationDisabled()) { - - uassert(ErrorCodes::BadValue, - str::stream() << "New document and old document both need to have " << kSafeContent - << " field.", - compareSafeContentElem(oldDoc.value(), newDoc)); - } - dassert(opCtx->lockState()->isCollectionLockedForMode(ns(), MODE_IX)); invariant(oldDoc.snapshotId() == opCtx->recoveryUnit()->getSnapshotId()); invariant(newDoc.isOwned()); diff --git a/src/mongo/db/catalog/document_validation.h b/src/mongo/db/catalog/document_validation.h index 47db304d79d..875f255c565 100644 --- a/src/mongo/db/catalog/document_validation.h +++ b/src/mongo/db/catalog/document_validation.h @@ -52,7 +52,7 @@ class DocumentValidationSettings { public: enum flag : std::uint8_t { /* - * Enables document validation (schema, internal, and safeContent). + * Enables document validation (both schema and internal). */ kEnableValidation = 0x00, /* @@ -67,12 +67,6 @@ public: * doesn't comply with internal validation rules. */ kDisableInternalValidation = 0x02, - /* - * If set, modifications to the safeContent array are allowed. This flag is only - * enabled when bypass document validation is enabled or if crudProcessed is true - * in the query. - */ - kDisableSafeContentValidation = 0x04, }; using Flags = std::uint8_t; @@ -98,10 +92,6 @@ public: return _flags & kDisableInternalValidation; } - bool isSafeContentValidationDisabled() const { - return _flags & kDisableSafeContentValidation; - } - bool isDocumentValidationEnabled() const { return _flags == kEnableValidation; } @@ -144,29 +134,11 @@ class DisableDocumentSchemaValidationIfTrue { public: DisableDocumentSchemaValidationIfTrue(OperationContext* opCtx, bool shouldDisableSchemaValidation) { - if (shouldDisableSchemaValidation) { - _documentSchemaValidationDisabler.emplace( - opCtx, DocumentValidationSettings::kDisableSchemaValidation); - } - } - -private: - boost::optional<DisableDocumentValidation> _documentSchemaValidationDisabler; -}; - -class DisableSafeContentValidationIfTrue { -public: - DisableSafeContentValidationIfTrue(OperationContext* opCtx, - bool shouldDisableSchemaValidation, - bool encryptionInformationCrudProcessed) { - if (shouldDisableSchemaValidation || encryptionInformationCrudProcessed) { - _documentSchemaValidationDisabler.emplace( - opCtx, DocumentValidationSettings::kDisableSafeContentValidation); - } + if (shouldDisableSchemaValidation) + _documentSchemaValidationDisabler.emplace(opCtx); } private: boost::optional<DisableDocumentValidation> _documentSchemaValidationDisabler; }; - } // namespace mongo diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index abbc0d834fd..00203a4c485 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -636,15 +636,10 @@ write_ops::FindAndModifyCommandReply CmdFindAndModify::Invocation::typedRun( // Collect metrics. CmdFindAndModify::collectMetrics(req); - auto disableDocumentValidation = req.getBypassDocumentValidation().value_or(false); - auto fleCrudProcessed = - write_ops_exec::getFleCrudProcessed(opCtx, req.getEncryptionInformation()); - - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); - - DisableSafeContentValidationIfTrue safeContentValidationDisabler( - opCtx, disableDocumentValidation, fleCrudProcessed); + boost::optional<DisableDocumentValidation> maybeDisableValidation; + if (req.getBypassDocumentValidation().value_or(false)) { + maybeDisableValidation.emplace(opCtx); + } const auto inTransaction = opCtx->inMultiDocumentTransaction(); uassert(50781, diff --git a/src/mongo/db/commands/fle_compact_test.cpp b/src/mongo/db/commands/fle_compact_test.cpp index 26153aadcc8..18c52f548ef 100644 --- a/src/mongo/db/commands/fle_compact_test.cpp +++ b/src/mongo/db/commands/fle_compact_test.cpp @@ -395,13 +395,8 @@ void FleCompactTest::doSingleInsert(int id, BSONObj encryptedFieldsObj) { auto efc = generateEncryptedFieldConfig(encryptedFieldsObj.getFieldNames<std::set<std::string>>()); - uassertStatusOK(processInsert(_queryImpl.get(), - _namespaces.edcNss, - serverPayload, - efc, - kUninitializedTxnNumber, - result, - false)); + uassertStatusOK(processInsert( + _queryImpl.get(), _namespaces.edcNss, serverPayload, efc, kUninitializedTxnNumber, result)); } void FleCompactTest::doSingleDelete(int id, BSONObj encryptedFieldsObj) { diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index 73933d1abe2..c63a2978e11 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -66,6 +66,7 @@ #include "mongo/db/repl/tenant_migration_recipient_service.h" #include "mongo/db/s/active_migrations_registry.h" #include "mongo/db/s/balancer/balancer.h" +#include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/config/configsvr_coordinator_service.h" #include "mongo/db/s/config/sharding_catalog_manager.h" #include "mongo/db/s/migration_coordinator_document_gen.h" @@ -74,11 +75,11 @@ #include "mongo/db/s/resharding/coordinator_document_gen.h" #include "mongo/db/s/resharding/resharding_coordinator_service.h" #include "mongo/db/s/resharding/resharding_donor_recipient_common.h" -#include "mongo/db/s/shard_filtering_metadata_refresh.h" +#include "mongo/db/s/shard_metadata_util.h" #include "mongo/db/s/sharding_ddl_coordinator_service.h" -#include "mongo/db/s/sharding_state.h" #include "mongo/db/s/sharding_util.h" #include "mongo/db/s/transaction_coordinator_service.h" +#include "mongo/db/s/type_shard_collection.h" #include "mongo/db/server_feature_flags_gen.h" #include "mongo/db/server_options.h" #include "mongo/db/session_catalog.h" @@ -90,7 +91,6 @@ #include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/s/catalog/sharding_catalog_client.h" -#include "mongo/s/catalog_cache_loader.h" #include "mongo/s/pm2423_feature_flags_gen.h" #include "mongo/s/pm2583_feature_flags_gen.h" #include "mongo/s/refine_collection_shard_key_coordinator_feature_flags_gen.h" @@ -225,6 +225,20 @@ void uassertStatusOKIgnoreNSNotFound(Status status) { uassertStatusOK(status); } +void clearFilteringMetadataOnSecondaries(OperationContext* opCtx, const NamespaceString& collName) { + Status signalStatus = shardmetadatautil::updateShardCollectionsEntry( + opCtx, + BSON(ShardCollectionType::kNssFieldName << collName.ns()), + BSON("$inc" << BSON(ShardCollectionType::kEnterCriticalSectionCounterFieldName << 1)), + false /*upsert*/); + + uassertStatusOKWithContext( + signalStatus, + str::stream() + << "Failed to persist signal to clear the filtering metadata on secondaries for nss " + << collName.ns()); +} + /** * Sets the minimum allowed feature compatibility version for the cluster. The cluster should not * use any new features introduced in binary versions that are newer than the feature compatibility @@ -550,18 +564,26 @@ public: } } - if (requestedVersion == multiversion::FeatureCompatibilityVersion::kVersion_6_0 && - ShardingState::get(opCtx)->enabled()) { + if (requestedVersion == multiversion::FeatureCompatibilityVersion::kVersion_5_3 || + requestedVersion == multiversion::FeatureCompatibilityVersion::kVersion_6_0) { const auto colls = CollectionShardingState::getCollectionNames(opCtx); for (const auto& collName : colls) { - onShardVersionMismatch(opCtx, collName, boost::none); - CatalogCacheLoader::get(opCtx).waitForCollectionFlush(opCtx, collName); + try { + if (!collName.isSystemDotViews()) { + { + AutoGetCollection coll(opCtx, collName, MODE_IX); + CollectionShardingState::get(opCtx, collName) + ->clearFilteringMetadata_DoNotUseIt(opCtx); + } + clearFilteringMetadataOnSecondaries(opCtx, collName); + } + } catch (const ExceptionFor<ErrorCodes::CommandNotSupportedOnView>&) { + // Nothing to do since collName is a view + } } - repl::ReplClientInfo::forClient(opCtx->getClient()) - .setLastOpToSystemLastOpTime(opCtx); - - // Wait until the changes on config.cache.* are majority committed. + // Wait until the signals to clear the filtering metadata on secondary nodes are + // majority committed. WriteConcernResult ignoreResult; auto latestOpTime = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); uassertStatusOK(waitForWriteConcern(opCtx, diff --git a/src/mongo/db/commands/write_commands.cpp b/src/mongo/db/commands/write_commands.cpp index 0254baca47d..9e6d189b4a3 100644 --- a/src/mongo/db/commands/write_commands.cpp +++ b/src/mongo/db/commands/write_commands.cpp @@ -529,8 +529,7 @@ public: write_ops::InsertCommandReply typedRun(OperationContext* opCtx) final try { transactionChecks(opCtx, ns()); - if (request().getEncryptionInformation().has_value() && - !request().getEncryptionInformation()->getCrudProcessed()) { + if (request().getEncryptionInformation().has_value()) { write_ops::InsertCommandReply insertReply; auto batch = processFLEInsert(opCtx, request(), &insertReply); if (batch == FLEBatchResult::kProcessed) { @@ -1457,8 +1456,7 @@ public: write_ops::UpdateCommandReply updateReply; OperationSource source = OperationSource::kStandard; - if (request().getEncryptionInformation().has_value() && - !request().getEncryptionInformation().get().getCrudProcessed()) { + if (request().getEncryptionInformation().has_value()) { return processFLEUpdate(opCtx, request()); } diff --git a/src/mongo/db/dbmessage.h b/src/mongo/db/dbmessage.h index e280bf296be..95e0cfc6f7f 100644 --- a/src/mongo/db/dbmessage.h +++ b/src/mongo/db/dbmessage.h @@ -227,7 +227,7 @@ public: * Indicates whether this message is expected to have a ns. */ bool messageShouldHaveNs() const { - return static_cast<int>(_msg.operation() >= dbUpdate) & (_msg.operation() <= dbDelete); + return (_msg.operation() >= dbUpdate) & (_msg.operation() <= dbDelete); } /** diff --git a/src/mongo/db/exec/add_fields_projection_executor.cpp b/src/mongo/db/exec/add_fields_projection_executor.cpp index a0fd7f08580..592074b4834 100644 --- a/src/mongo/db/exec/add_fields_projection_executor.cpp +++ b/src/mongo/db/exec/add_fields_projection_executor.cpp @@ -92,6 +92,38 @@ private: // The original object. Used to generate more helpful error messages. const BSONObj& _rawObj; + // Custom comparator that orders fieldpath strings by path prefix first, then by field. + struct PathPrefixComparator { + static constexpr char dot = '.'; + + // Returns true if the lhs value should sort before the rhs, false otherwise. + bool operator()(const std::string& lhs, const std::string& rhs) const { + for (size_t pos = 0, len = std::min(lhs.size(), rhs.size()); pos < len; ++pos) { + auto &lchar = lhs[pos], &rchar = rhs[pos]; + if (lchar == rchar) { + continue; + } + + // Consider the path delimiter '.' as being less than all other characters, so that + // paths sort directly before any paths they prefix and directly after any paths + // which prefix them. + if (lchar == dot) { + return true; + } else if (rchar == dot) { + return false; + } + + // Otherwise, default to normal character comparison. + return lchar < rchar; + } + + // If we get here, then we have reached the end of lhs and/or rhs and all of their path + // segments up to this point match. If lhs is shorter than rhs, then lhs prefixes rhs + // and should sort before it. + return lhs.size() < rhs.size(); + } + }; + // Tracks which paths we've seen to ensure no two paths conflict with each other. std::set<std::string, PathPrefixComparator> _seenPaths; }; diff --git a/src/mongo/db/fle_crud.cpp b/src/mongo/db/fle_crud.cpp index 58136f31c44..a847304b55f 100644 --- a/src/mongo/db/fle_crud.cpp +++ b/src/mongo/db/fle_crud.cpp @@ -190,20 +190,16 @@ std::pair<FLEBatchResult, write_ops::InsertCommandReply> processInsert( auto edcNss = insertRequest.getNamespace(); auto ei = insertRequest.getEncryptionInformation().get(); - bool bypassDocumentValidation = - insertRequest.getWriteCommandRequestBase().getBypassDocumentValidation(); - auto efc = EncryptionInformationHelpers::getAndValidateSchema(edcNss, ei); auto documents = insertRequest.getDocuments(); // TODO - how to check if a document will be too large??? - uassert(6371202, "Only single insert batches are supported in Queryable Encryption", documents.size() == 1); auto document = documents[0]; - EDCServerCollection::validateEncryptedFieldInfo(document, efc, bypassDocumentValidation); + EDCServerCollection::validateEncryptedFieldInfo(document, efc); auto serverPayload = std::make_shared<std::vector<EDCServerPayloadInfo>>( EDCServerCollection::getEncryptedFieldInfo(document)); @@ -227,8 +223,8 @@ std::pair<FLEBatchResult, write_ops::InsertCommandReply> processInsert( auto swResult = trun->runNoThrow( opCtx, - [sharedInsertBlock, reply, ownedDocument, bypassDocumentValidation]( - const txn_api::TransactionClient& txnClient, ExecutorPtr txnExec) { + [sharedInsertBlock, reply, ownedDocument](const txn_api::TransactionClient& txnClient, + ExecutorPtr txnExec) { FLEQueryInterfaceImpl queryImpl(txnClient, getGlobalServiceContext()); auto [edcNss2, efc2, serverPayload2, stmtId2] = *sharedInsertBlock.get(); @@ -238,13 +234,8 @@ std::pair<FLEBatchResult, write_ops::InsertCommandReply> processInsert( fleCrudHangPreInsert.pauseWhileSet(); } - *reply = uassertStatusOK(processInsert(&queryImpl, - edcNss2, - *serverPayload2.get(), - efc2, - stmtId2, - ownedDocument, - bypassDocumentValidation)); + *reply = uassertStatusOK(processInsert( + &queryImpl, edcNss2, *serverPayload2.get(), efc2, stmtId2, ownedDocument)); if (MONGO_unlikely(fleCrudHangInsert.shouldFail())) { LOGV2(6371903, "Hanging due to fleCrudHangInsert fail point"); @@ -450,8 +441,7 @@ void processFieldsForInsert(FLEQueryInterface* queryImpl, const NamespaceString& edcNss, std::vector<EDCServerPayloadInfo>& serverPayload, const EncryptedFieldConfig& efc, - int32_t* pStmtId, - bool bypassDocumentValidation) { + int32_t* pStmtId) { NamespaceString nssEsc(edcNss.db(), efc.getEscCollection().get()); @@ -519,8 +509,7 @@ void processFieldsForInsert(FLEQueryInterface* queryImpl, ECOCCollection::generateDocument(payload.fieldPathName, payload.payload.getEncryptedTokens()), pStmtId, - false, - bypassDocumentValidation)); + false)); checkWriteErrors(ecocInsertReply); } } @@ -730,11 +719,9 @@ StatusWith<write_ops::InsertCommandReply> processInsert( std::vector<EDCServerPayloadInfo>& serverPayload, const EncryptedFieldConfig& efc, int32_t stmtId, - BSONObj document, - bool bypassDocumentValidation) { + BSONObj document) { - processFieldsForInsert( - queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); + processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); auto finalDoc = EDCServerCollection::finalizeForInsert(document, serverPayload); @@ -805,9 +792,6 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, auto tokenMap = EncryptionInformationHelpers::getDeleteTokens(edcNss, ei); const auto updateOpEntry = updateRequest.getUpdates()[0]; - auto bypassDocumentValidation = - updateRequest.getWriteCommandRequestBase().getBypassDocumentValidation(); - const auto updateModification = updateOpEntry.getU(); int32_t stmtId = getStmtIdForWriteAt(updateRequest, 0); @@ -815,26 +799,16 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, // Step 1 ---- std::vector<EDCServerPayloadInfo> serverPayload; auto newUpdateOpEntry = updateRequest.getUpdates()[0]; - - auto highCardinalityModeAllowed = newUpdateOpEntry.getUpsert() - ? fle::HighCardinalityModeAllowed::kDisallow - : fle::HighCardinalityModeAllowed::kAllow; - - newUpdateOpEntry.setQ(fle::rewriteEncryptedFilterInsideTxn(queryImpl, - updateRequest.getDbName(), - efc, - expCtx, - newUpdateOpEntry.getQ(), - highCardinalityModeAllowed)); + newUpdateOpEntry.setQ(fle::rewriteEncryptedFilterInsideTxn( + queryImpl, updateRequest.getDbName(), efc, expCtx, newUpdateOpEntry.getQ())); if (updateModification.type() == write_ops::UpdateModification::Type::kModifier) { auto updateModifier = updateModification.getUpdateModifier(); auto setObject = updateModifier.getObjectField("$set"); - EDCServerCollection::validateEncryptedFieldInfo(setObject, efc, bypassDocumentValidation); + EDCServerCollection::validateEncryptedFieldInfo(setObject, efc); serverPayload = EDCServerCollection::getEncryptedFieldInfo(updateModifier); - processFieldsForInsert( - queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); + processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); // Step 2 ---- auto pushUpdate = EDCServerCollection::finalizeForUpdate(updateModifier, serverPayload); @@ -843,12 +817,10 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, pushUpdate, write_ops::UpdateModification::ClassicTag(), false)); } else { auto replacementDocument = updateModification.getUpdateReplacement(); - EDCServerCollection::validateEncryptedFieldInfo( - replacementDocument, efc, bypassDocumentValidation); + EDCServerCollection::validateEncryptedFieldInfo(replacementDocument, efc); serverPayload = EDCServerCollection::getEncryptedFieldInfo(replacementDocument); - processFieldsForInsert( - queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); + processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); // Step 2 ---- auto safeContentReplace = @@ -863,8 +835,6 @@ write_ops::UpdateCommandReply processUpdate(FLEQueryInterface* queryImpl, newUpdateRequest.setUpdates({newUpdateOpEntry}); newUpdateRequest.getWriteCommandRequestBase().setStmtIds(boost::none); newUpdateRequest.getWriteCommandRequestBase().setStmtId(stmtId); - newUpdateRequest.getWriteCommandRequestBase().setBypassDocumentValidation( - bypassDocumentValidation); ++stmtId; auto [updateReply, originalDocument] = @@ -922,10 +892,6 @@ FLEBatchResult processFLEBatch(OperationContext* opCtx, BatchedCommandResponse* response, boost::optional<OID> targetEpoch) { - if (request.getWriteCommandRequestBase().getEncryptionInformation()->getCrudProcessed()) { - return FLEBatchResult::kNotProcessed; - } - // TODO (SERVER-65077): Remove FCV check once 6.0 is released uassert(6371209, "Queryable Encryption is only supported when FCV supports 6.0", @@ -1004,25 +970,19 @@ std::unique_ptr<BatchedCommandRequest> processFLEBatchExplain( request.getNS(), deleteRequest.getEncryptionInformation().get(), newDeleteOp.getQ(), - &getTransactionWithRetriesForMongoS, - fle::HighCardinalityModeAllowed::kAllow)); + &getTransactionWithRetriesForMongoS)); deleteRequest.setDeletes({newDeleteOp}); deleteRequest.getWriteCommandRequestBase().setEncryptionInformation(boost::none); return std::make_unique<BatchedCommandRequest>(deleteRequest); } else if (request.getBatchType() == BatchedCommandRequest::BatchType_Update) { auto updateRequest = request.getUpdateRequest(); auto newUpdateOp = updateRequest.getUpdates()[0]; - auto highCardinalityModeAllowed = newUpdateOp.getUpsert() - ? fle::HighCardinalityModeAllowed::kDisallow - : fle::HighCardinalityModeAllowed::kAllow; - newUpdateOp.setQ(fle::rewriteQuery(opCtx, getExpCtx(newUpdateOp), request.getNS(), updateRequest.getEncryptionInformation().get(), newUpdateOp.getQ(), - &getTransactionWithRetriesForMongoS, - highCardinalityModeAllowed)); + &getTransactionWithRetriesForMongoS)); updateRequest.setUpdates({newUpdateOp}); updateRequest.getWriteCommandRequestBase().setEncryptionInformation(boost::none); return std::make_unique<BatchedCommandRequest>(updateRequest); @@ -1045,22 +1005,10 @@ write_ops::FindAndModifyCommandReply processFindAndModify( auto newFindAndModifyRequest = findAndModifyRequest; - const auto bypassDocumentValidation = - findAndModifyRequest.getBypassDocumentValidation().value_or(false); - // Step 0 ---- // Rewrite filter - auto highCardinalityModeAllowed = findAndModifyRequest.getUpsert().value_or(false) - ? fle::HighCardinalityModeAllowed::kDisallow - : fle::HighCardinalityModeAllowed::kAllow; - - newFindAndModifyRequest.setQuery( - fle::rewriteEncryptedFilterInsideTxn(queryImpl, - edcNss.db(), - efc, - expCtx, - findAndModifyRequest.getQuery(), - highCardinalityModeAllowed)); + newFindAndModifyRequest.setQuery(fle::rewriteEncryptedFilterInsideTxn( + queryImpl, edcNss.db(), efc, expCtx, findAndModifyRequest.getQuery())); // Make sure not to inherit the command's writeConcern, this should be set at the transaction // level. @@ -1077,11 +1025,9 @@ write_ops::FindAndModifyCommandReply processFindAndModify( if (updateModification.type() == write_ops::UpdateModification::Type::kModifier) { auto updateModifier = updateModification.getUpdateModifier(); auto setObject = updateModifier.getObjectField("$set"); - EDCServerCollection::validateEncryptedFieldInfo( - setObject, efc, bypassDocumentValidation); + EDCServerCollection::validateEncryptedFieldInfo(setObject, efc); serverPayload = EDCServerCollection::getEncryptedFieldInfo(updateModifier); - processFieldsForInsert( - queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); + processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); auto pushUpdate = EDCServerCollection::finalizeForUpdate(updateModifier, serverPayload); @@ -1090,12 +1036,10 @@ write_ops::FindAndModifyCommandReply processFindAndModify( pushUpdate, write_ops::UpdateModification::ClassicTag(), false); } else { auto replacementDocument = updateModification.getUpdateReplacement(); - EDCServerCollection::validateEncryptedFieldInfo( - replacementDocument, efc, bypassDocumentValidation); + EDCServerCollection::validateEncryptedFieldInfo(replacementDocument, efc); serverPayload = EDCServerCollection::getEncryptedFieldInfo(replacementDocument); - processFieldsForInsert( - queryImpl, edcNss, serverPayload, efc, &stmtId, bypassDocumentValidation); + processFieldsForInsert(queryImpl, edcNss, serverPayload, efc, &stmtId); // Step 2 ---- auto safeContentReplace = @@ -1187,17 +1131,8 @@ write_ops::FindAndModifyCommandRequest processFindAndModifyExplain( auto efc = EncryptionInformationHelpers::getAndValidateSchema(edcNss, ei); auto newFindAndModifyRequest = findAndModifyRequest; - auto highCardinalityModeAllowed = findAndModifyRequest.getUpsert().value_or(false) - ? fle::HighCardinalityModeAllowed::kDisallow - : fle::HighCardinalityModeAllowed::kAllow; - - newFindAndModifyRequest.setQuery( - fle::rewriteEncryptedFilterInsideTxn(queryImpl, - edcNss.db(), - efc, - expCtx, - findAndModifyRequest.getQuery(), - highCardinalityModeAllowed)); + newFindAndModifyRequest.setQuery(fle::rewriteEncryptedFilterInsideTxn( + queryImpl, edcNss.db(), efc, expCtx, findAndModifyRequest.getQuery())); newFindAndModifyRequest.setEncryptionInformation(boost::none); return newFindAndModifyRequest; @@ -1299,23 +1234,10 @@ uint64_t FLEQueryInterfaceImpl::countDocuments(const NamespaceString& nss) { } StatusWith<write_ops::InsertCommandReply> FLEQueryInterfaceImpl::insertDocument( - const NamespaceString& nss, - BSONObj obj, - StmtId* pStmtId, - bool translateDuplicateKey, - bool bypassDocumentValidation) { + const NamespaceString& nss, BSONObj obj, StmtId* pStmtId, bool translateDuplicateKey) { write_ops::InsertCommandRequest insertRequest(nss); insertRequest.setDocuments({obj}); - EncryptionInformation encryptionInformation; - encryptionInformation.setCrudProcessed(true); - - // We need to set an empty BSON object here for the schema. - encryptionInformation.setSchema(BSONObj()); - insertRequest.getWriteCommandRequestBase().setEncryptionInformation(encryptionInformation); - insertRequest.getWriteCommandRequestBase().setBypassDocumentValidation( - bypassDocumentValidation); - int32_t stmtId = *pStmtId; if (stmtId != kUninitializedStmtId) { (*pStmtId)++; @@ -1400,7 +1322,6 @@ std::pair<write_ops::UpdateCommandReply, BSONObj> FLEQueryInterfaceImpl::updateW findAndModifyRequest.setLet( mergeLetAndCVariables(updateRequest.getLet(), updateOpEntry.getC())); findAndModifyRequest.setStmtId(updateRequest.getStmtId()); - findAndModifyRequest.setBypassDocumentValidation(updateRequest.getBypassDocumentValidation()); auto ei2 = ei; ei2.setCrudProcessed(true); @@ -1442,15 +1363,9 @@ std::pair<write_ops::UpdateCommandReply, BSONObj> FLEQueryInterfaceImpl::updateW } write_ops::UpdateCommandReply FLEQueryInterfaceImpl::update( - const NamespaceString& nss, int32_t stmtId, write_ops::UpdateCommandRequest& updateRequest) { - - invariant(!updateRequest.getWriteCommandRequestBase().getEncryptionInformation()); - - EncryptionInformation encryptionInformation; - encryptionInformation.setCrudProcessed(true); - - encryptionInformation.setSchema(BSONObj()); - updateRequest.getWriteCommandRequestBase().setEncryptionInformation(encryptionInformation); + const NamespaceString& nss, + int32_t stmtId, + const write_ops::UpdateCommandRequest& updateRequest) { dassert(updateRequest.getStmtIds().value_or(std::vector<int32_t>()).empty()); diff --git a/src/mongo/db/fle_crud.h b/src/mongo/db/fle_crud.h index 7c8d93ae1f9..738e85b8996 100644 --- a/src/mongo/db/fle_crud.h +++ b/src/mongo/db/fle_crud.h @@ -261,11 +261,7 @@ public: * FLEStateCollectionContention instead. */ virtual StatusWith<write_ops::InsertCommandReply> insertDocument( - const NamespaceString& nss, - BSONObj obj, - StmtId* pStmtId, - bool translateDuplicateKey, - bool bypassDocumentValidation = false) = 0; + const NamespaceString& nss, BSONObj obj, StmtId* pStmtId, bool translateDuplicateKey) = 0; /** * Delete a single document with the given query. @@ -298,7 +294,7 @@ public: virtual write_ops::UpdateCommandReply update( const NamespaceString& nss, int32_t stmtId, - write_ops::UpdateCommandRequest& updateRequest) = 0; + const write_ops::UpdateCommandRequest& updateRequest) = 0; /** * Do a single findAndModify request. @@ -329,12 +325,10 @@ public: uint64_t countDocuments(const NamespaceString& nss) final; - StatusWith<write_ops::InsertCommandReply> insertDocument( - const NamespaceString& nss, - BSONObj obj, - int32_t* pStmtId, - bool translateDuplicateKey, - bool bypassDocumentValidation = false) final; + StatusWith<write_ops::InsertCommandReply> insertDocument(const NamespaceString& nss, + BSONObj obj, + int32_t* pStmtId, + bool translateDuplicateKey) final; std::pair<write_ops::DeleteCommandReply, BSONObj> deleteWithPreimage( const NamespaceString& nss, @@ -346,9 +340,10 @@ public: const EncryptionInformation& ei, const write_ops::UpdateCommandRequest& updateRequest) final; - write_ops::UpdateCommandReply update(const NamespaceString& nss, - int32_t stmtId, - write_ops::UpdateCommandRequest& updateRequest) final; + write_ops::UpdateCommandReply update( + const NamespaceString& nss, + int32_t stmtId, + const write_ops::UpdateCommandRequest& updateRequest) final; write_ops::FindAndModifyCommandReply findAndModify( const NamespaceString& nss, @@ -413,8 +408,7 @@ StatusWith<write_ops::InsertCommandReply> processInsert( std::vector<EDCServerPayloadInfo>& serverPayload, const EncryptedFieldConfig& efc, int32_t stmtId, - BSONObj document, - bool bypassDocumentValidation = false); + BSONObj document); /** * Process a FLE delete with the query interface diff --git a/src/mongo/db/fle_crud_mongod.cpp b/src/mongo/db/fle_crud_mongod.cpp index 1e488f1f65a..68327133c88 100644 --- a/src/mongo/db/fle_crud_mongod.cpp +++ b/src/mongo/db/fle_crud_mongod.cpp @@ -284,13 +284,7 @@ BSONObj processFLEWriteExplainD(OperationContext* opCtx, const BSONObj& query) { auto expCtx = make_intrusive<ExpressionContext>( opCtx, fle::collatorFromBSON(opCtx, collation), nss, runtimeConstants, letParameters); - return fle::rewriteQuery(opCtx, - expCtx, - nss, - info, - query, - &getTransactionWithRetriesForMongoD, - fle::HighCardinalityModeAllowed::kAllow); + return fle::rewriteQuery(opCtx, expCtx, nss, info, query, &getTransactionWithRetriesForMongoD); } std::pair<write_ops::FindAndModifyCommandRequest, OpMsgRequest> diff --git a/src/mongo/db/fle_crud_test.cpp b/src/mongo/db/fle_crud_test.cpp index 0a5d7dfc37c..527dd5bca11 100644 --- a/src/mongo/db/fle_crud_test.cpp +++ b/src/mongo/db/fle_crud_test.cpp @@ -27,7 +27,6 @@ * it in the license file. */ -#include "mongo/base/error_codes.h" #include "mongo/platform/basic.h" #include <algorithm> @@ -154,12 +153,8 @@ protected: void assertDocumentCounts(uint64_t edc, uint64_t esc, uint64_t ecc, uint64_t ecoc); - void testValidateEncryptedFieldInfo(BSONObj obj, bool bypassValidation); - - void testValidateTags(BSONObj obj); - - void doSingleInsert(int id, BSONElement element, bool bypassDocumentValidation = false); - void doSingleInsert(int id, BSONObj obj, bool bypassDocumentValidation = false); + void doSingleInsert(int id, BSONElement element); + void doSingleInsert(int id, BSONObj obj); void doSingleInsertWithContention( int id, BSONElement element, int64_t cm, uint64_t cf, EncryptedFieldConfig efc); @@ -411,7 +406,7 @@ void FleCrudTest::doSingleWideInsert(int id, uint64_t fieldCount, ValueGenerator auto efc = getTestEncryptedFieldConfig(); - uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result, false)); + uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result)); } @@ -456,16 +451,7 @@ std::vector<char> generateSinglePlaceholder(BSONElement value, int64_t cm = 0) { return v; } -void FleCrudTest::testValidateEncryptedFieldInfo(BSONObj obj, bool bypassValidation) { - auto efc = getTestEncryptedFieldConfig(); - EDCServerCollection::validateEncryptedFieldInfo(obj, efc, bypassValidation); -} - -void FleCrudTest::testValidateTags(BSONObj obj) { - FLEClientCrypto::validateTagsArray(obj); -} - -void FleCrudTest::doSingleInsert(int id, BSONElement element, bool bypassDocumentValidation) { +void FleCrudTest::doSingleInsert(int id, BSONElement element) { auto buf = generateSinglePlaceholder(element); BSONObjBuilder builder; builder.append("_id", id); @@ -481,10 +467,10 @@ void FleCrudTest::doSingleInsert(int id, BSONElement element, bool bypassDocumen auto efc = getTestEncryptedFieldConfig(); - uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result, false)); + uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result)); } -void FleCrudTest::doSingleInsert(int id, BSONObj obj, bool bypassDocumentValidation) { +void FleCrudTest::doSingleInsert(int id, BSONObj obj) { doSingleInsert(id, obj.firstElement()); } @@ -504,7 +490,7 @@ void FleCrudTest::doSingleInsertWithContention( auto serverPayload = EDCServerCollection::getEncryptedFieldInfo(result); - uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result, false)); + uassertStatusOK(processInsert(_queryImpl.get(), _edcNs, serverPayload, efc, 0, result)); } void FleCrudTest::doSingleInsertWithContention( @@ -904,6 +890,7 @@ TEST_F(FleCrudTest, UpdateOneSameValue) { << "secret")); } + // Update one document with replacement TEST_F(FleCrudTest, UpdateOneReplace) { @@ -969,16 +956,7 @@ TEST_F(FleCrudTest, SetSafeContent) { builder.append("$set", BSON(kSafeContent << "foo")); auto result = builder.obj(); - ASSERT_THROWS_CODE(doSingleUpdateWithUpdateDoc(1, result), DBException, 6666200); -} - -// Test that EDCServerCollection::validateEncryptedFieldInfo checks that the -// safeContent cannot be present in the BSON obj. -TEST_F(FleCrudTest, testValidateEncryptedFieldConfig) { - testValidateEncryptedFieldInfo(BSON(kSafeContent << "secret"), true); - ASSERT_THROWS_CODE(testValidateEncryptedFieldInfo(BSON(kSafeContent << "secret"), false), - DBException, - 6666200); + ASSERT_THROWS_CODE(doSingleUpdateWithUpdateDoc(1, result), DBException, 6371507); } // Update one document via findAndModify @@ -1060,11 +1038,6 @@ TEST_F(FleCrudTest, FindAndModify_RenameSafeContent) { ASSERT_THROWS_CODE(doFindAndModify(req), DBException, 6371506); } -TEST_F(FleCrudTest, validateTagsTest) { - testValidateTags(BSON(kSafeContent << BSON_ARRAY(123))); - ASSERT_THROWS_CODE(testValidateTags(BSON(kSafeContent << "foo")), DBException, 6371507); -} - // Mess with __safeContent__ and ensure the update errors TEST_F(FleCrudTest, FindAndModify_SetSafeContent) { doSingleInsert(1, @@ -1083,7 +1056,8 @@ TEST_F(FleCrudTest, FindAndModify_SetSafeContent) { req.setUpdate( write_ops::UpdateModification(result, write_ops::UpdateModification::ClassicTag{}, false)); - ASSERT_THROWS_CODE(doFindAndModify(req), DBException, 6666200); + + ASSERT_THROWS_CODE(doFindAndModify(req), DBException, 6371507); } TEST_F(FleTagsTest, InsertOne) { @@ -1225,7 +1199,7 @@ TEST_F(FleTagsTest, MemoryLimit) { doSingleInsert(10, doc); // readTags returns 11 tags which does exceed memory limit. - ASSERT_THROWS_CODE(readTags(doc), DBException, ErrorCodes::FLEMaxTagLimitExceeded); + ASSERT_THROWS_CODE(readTags(doc), DBException, 6401800); doSingleDelete(5); diff --git a/src/mongo/db/fle_query_interface_mock.cpp b/src/mongo/db/fle_query_interface_mock.cpp index b5ca4e1e9cd..2aeb39788dd 100644 --- a/src/mongo/db/fle_query_interface_mock.cpp +++ b/src/mongo/db/fle_query_interface_mock.cpp @@ -54,11 +54,7 @@ uint64_t FLEQueryInterfaceMock::countDocuments(const NamespaceString& nss) { } StatusWith<write_ops::InsertCommandReply> FLEQueryInterfaceMock::insertDocument( - const NamespaceString& nss, - BSONObj obj, - StmtId* pStmtId, - bool translateDuplicateKey, - bool bypassDocumentValidation) { + const NamespaceString& nss, BSONObj obj, StmtId* pStmtId, bool translateDuplicateKey) { repl::TimestampedBSONObj tb; tb.obj = obj; @@ -136,7 +132,9 @@ std::pair<write_ops::UpdateCommandReply, BSONObj> FLEQueryInterfaceMock::updateW } write_ops::UpdateCommandReply FLEQueryInterfaceMock::update( - const NamespaceString& nss, int32_t stmtId, write_ops::UpdateCommandRequest& updateRequest) { + const NamespaceString& nss, + int32_t stmtId, + const write_ops::UpdateCommandRequest& updateRequest) { auto [reply, _] = updateWithPreimage(nss, EncryptionInformation(), updateRequest); return reply; } diff --git a/src/mongo/db/fle_query_interface_mock.h b/src/mongo/db/fle_query_interface_mock.h index a89fc71ce1e..229d2c08dfe 100644 --- a/src/mongo/db/fle_query_interface_mock.h +++ b/src/mongo/db/fle_query_interface_mock.h @@ -47,12 +47,10 @@ public: uint64_t countDocuments(const NamespaceString& nss) final; - StatusWith<write_ops::InsertCommandReply> insertDocument( - const NamespaceString& nss, - BSONObj obj, - StmtId* pStmtId, - bool translateDuplicateKey, - bool bypassDocumentValidation = false) final; + StatusWith<write_ops::InsertCommandReply> insertDocument(const NamespaceString& nss, + BSONObj obj, + StmtId* pStmtId, + bool translateDuplicateKey) final; std::pair<write_ops::DeleteCommandReply, BSONObj> deleteWithPreimage( const NamespaceString& nss, @@ -64,9 +62,10 @@ public: const EncryptionInformation& ei, const write_ops::UpdateCommandRequest& updateRequest) final; - write_ops::UpdateCommandReply update(const NamespaceString& nss, - int32_t stmtId, - write_ops::UpdateCommandRequest& updateRequest) final; + write_ops::UpdateCommandReply update( + const NamespaceString& nss, + int32_t stmtId, + const write_ops::UpdateCommandRequest& updateRequest) final; write_ops::FindAndModifyCommandReply findAndModify( const NamespaceString& nss, diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp index 45c1afefdae..1e06ec2932a 100644 --- a/src/mongo/db/index_builds_coordinator.cpp +++ b/src/mongo/db/index_builds_coordinator.cpp @@ -29,9 +29,9 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage -#include "mongo/db/index_builds_coordinator.h" +#include "mongo/platform/basic.h" -#include <fmt/format.h> +#include "mongo/db/index_builds_coordinator.h" #include "mongo/db/catalog/clustered_collection_util.h" #include "mongo/db/catalog/collection_catalog.h" @@ -1659,39 +1659,19 @@ void IndexBuildsCoordinator::assertNoIndexBuildInProgress() const { void IndexBuildsCoordinator::assertNoIndexBuildInProgForCollection( const UUID& collectionUUID) const { - boost::optional<UUID> firstIndexBuildUUID; - auto indexBuilds = activeIndexBuilds.filterIndexBuilds([&](const auto& replState) { - auto isIndexBuildForCollection = (collectionUUID == replState.collectionUUID); - if (isIndexBuildForCollection && !firstIndexBuildUUID) { - firstIndexBuildUUID = replState.buildUUID; - }; - return isIndexBuildForCollection; - }); - uassert(ErrorCodes::BackgroundOperationInProgressForNamespace, - fmt::format("cannot perform operation: an index build is currently running for " - "collection with UUID: {}. Found index build: {}", - collectionUUID.toString(), - firstIndexBuildUUID->toString()), - indexBuilds.empty()); + str::stream() << "cannot perform operation: an index build is currently running for " + "collection with UUID: " + << collectionUUID, + !inProgForCollection(collectionUUID)); } void IndexBuildsCoordinator::assertNoBgOpInProgForDb(StringData db) const { - boost::optional<UUID> firstIndexBuildUUID; - auto indexBuilds = activeIndexBuilds.filterIndexBuilds([&](const auto& replState) { - auto isIndexBuildForCollection = (db == replState.dbName); - if (isIndexBuildForCollection && !firstIndexBuildUUID) { - firstIndexBuildUUID = replState.buildUUID; - }; - return isIndexBuildForCollection; - }); - uassert(ErrorCodes::BackgroundOperationInProgressForDatabase, - fmt::format("cannot perform operation: an index build is currently running for " - "database {}. Found index build: {}", - db, - firstIndexBuildUUID->toString()), - indexBuilds.empty()); + str::stream() << "cannot perform operation: an index build is currently running for " + "database " + << db, + !inProgForDb(db)); } void IndexBuildsCoordinator::awaitNoIndexBuildInProgressForCollection(OperationContext* opCtx, diff --git a/src/mongo/db/index_builds_coordinator_mongod_test.cpp b/src/mongo/db/index_builds_coordinator_mongod_test.cpp index ecca38a9d94..28e09af1ef7 100644 --- a/src/mongo/db/index_builds_coordinator_mongod_test.cpp +++ b/src/mongo/db/index_builds_coordinator_mongod_test.cpp @@ -86,9 +86,6 @@ void IndexBuildsCoordinatorMongodTest::setUp() { } void IndexBuildsCoordinatorMongodTest::tearDown() { - // Resume index builds left running by test failures so that shutdown() will not block. - _indexBuildsCoord->sleepIndexBuilds_forTestOnly(false); - _indexBuildsCoord->shutdown(operationContext()); _indexBuildsCoord.reset(); // All databases are dropped during tear down. @@ -158,27 +155,24 @@ TEST_F(IndexBuildsCoordinatorMongodTest, Registration) { _indexBuildsCoord->sleepIndexBuilds_forTestOnly(true); // Register an index build on _testFooNss. - auto testFoo1BuildUUID = UUID::gen(); auto testFoo1Future = assertGet(_indexBuildsCoord->startIndexBuild(operationContext(), _testFooNss.db().toString(), _testFooUUID, makeSpecs(_testFooNss, {"a", "b"}), - testFoo1BuildUUID, + UUID::gen(), IndexBuildProtocol::kTwoPhase, _indexBuildOptions)); ASSERT_EQ(_indexBuildsCoord->numInProgForDb(_testFooNss.db()), 1); ASSERT(_indexBuildsCoord->inProgForCollection(_testFooUUID)); ASSERT(_indexBuildsCoord->inProgForDb(_testFooNss.db())); - ASSERT_THROWS_WITH_CHECK( - _indexBuildsCoord->assertNoIndexBuildInProgForCollection(_testFooUUID), - ExceptionFor<ErrorCodes::BackgroundOperationInProgressForNamespace>, - [&](const auto& ex) { ASSERT_STRING_CONTAINS(ex.reason(), testFoo1BuildUUID.toString()); }); - ASSERT_THROWS_WITH_CHECK( - _indexBuildsCoord->assertNoBgOpInProgForDb(_testFooNss.db()), - ExceptionFor<ErrorCodes::BackgroundOperationInProgressForDatabase>, - [&](const auto& ex) { ASSERT_STRING_CONTAINS(ex.reason(), testFoo1BuildUUID.toString()); }); + ASSERT_THROWS_CODE(_indexBuildsCoord->assertNoIndexBuildInProgForCollection(_testFooUUID), + AssertionException, + ErrorCodes::BackgroundOperationInProgressForNamespace); + ASSERT_THROWS_CODE(_indexBuildsCoord->assertNoBgOpInProgForDb(_testFooNss.db()), + AssertionException, + ErrorCodes::BackgroundOperationInProgressForDatabase); // Register a second index build on _testFooNss. auto testFoo2Future = @@ -388,10 +382,7 @@ TEST_F(IndexBuildsCoordinatorMongodTest, AbortBuildIndexDueToTenantMigration) { // we currently have one index build in progress. ASSERT_EQ(1, _indexBuildsCoord->getActiveIndexBuildCount(operationContext())); - ASSERT_THROWS_WITH_CHECK( - _indexBuildsCoord->assertNoIndexBuildInProgress(), - ExceptionFor<ErrorCodes::BackgroundOperationInProgressForDatabase>, - [&](const auto& ex) { ASSERT_STRING_CONTAINS(ex.reason(), buildUUID.toString()); }); + ASSERT_THROWS(_indexBuildsCoord->assertNoIndexBuildInProgress(), mongo::DBException); ASSERT_OK(_indexBuildsCoord->voteCommitIndexBuild( operationContext(), buildUUID, HostAndPort("test1", 1234))); diff --git a/src/mongo/db/internal_transactions_feature_flag.idl b/src/mongo/db/internal_transactions_feature_flag.idl index bbbb9fa1477..d0373f56140 100644 --- a/src/mongo/db/internal_transactions_feature_flag.idl +++ b/src/mongo/db/internal_transactions_feature_flag.idl @@ -41,11 +41,6 @@ feature_flags: default: true version: 6.0 - featureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp: - description: Feature flag to enable always creating the config.transactions partial index on step up to primary even if the collection is not empty. - cpp_varname: gFeatureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp - default: false - featureFlagUpdateDocumentShardKeyUsingTransactionApi: description: Feature flag to enable usage of the transaction api for update findAndModify and update commands that change a document's shard key. cpp_varname: gFeatureFlagUpdateDocumentShardKeyUsingTransactionApi diff --git a/src/mongo/db/ops/write_ops_exec.cpp b/src/mongo/db/ops/write_ops_exec.cpp index f7fc2a84efd..6fd7e2200c7 100644 --- a/src/mongo/db/ops/write_ops_exec.cpp +++ b/src/mongo/db/ops/write_ops_exec.cpp @@ -602,36 +602,11 @@ SingleWriteResult makeWriteResultForInsertOrDeleteRetry() { return res; } - -// Returns the flags that determine the type of document validation we want to -// perform. First item in the tuple determines whether to bypass document validation altogether, -// second item determines if _safeContent_ array can be modified in an encrypted collection. -std::tuple<bool, bool> getDocumentValidationFlags(OperationContext* opCtx, - const write_ops::WriteCommandRequestBase& req) { - auto& encryptionInfo = req.getEncryptionInformation(); - const bool fleCrudProcessed = getFleCrudProcessed(opCtx, encryptionInfo); - return std::make_tuple(req.getBypassDocumentValidation(), fleCrudProcessed); -} } // namespace -bool getFleCrudProcessed(OperationContext* opCtx, - const boost::optional<EncryptionInformation>& encryptionInfo) { - if (encryptionInfo && encryptionInfo->getCrudProcessed().value_or(false)) { - uassert(6666201, - "External users cannot have crudProcessed enabled", - AuthorizationSession::get(opCtx->getClient()) - ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), - ActionType::internal)); - - return true; - } - return false; -} - WriteResult performInserts(OperationContext* opCtx, const write_ops::InsertCommandRequest& wholeOp, OperationSource source) { - // Insert performs its own retries, so we should only be within a WriteUnitOfWork when run in a // transaction. auto txnParticipant = TransactionParticipant::get(opCtx); @@ -666,15 +641,8 @@ WriteResult performInserts(OperationContext* opCtx, uassertStatusOK(userAllowedWriteNS(opCtx, wholeOp.getNamespace())); } - const auto [disableDocumentValidation, fleCrudProcessed] = - getDocumentValidationFlags(opCtx, wholeOp.getWriteCommandRequestBase()); - - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); - - DisableSafeContentValidationIfTrue safeContentValidationDisabler( - opCtx, disableDocumentValidation, fleCrudProcessed); - + DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler( + opCtx, wholeOp.getWriteCommandRequestBase().getBypassDocumentValidation()); LastOpFixer lastOpFixer(opCtx, wholeOp.getNamespace()); WriteResult out; @@ -1032,15 +1000,8 @@ WriteResult performUpdates(OperationContext* opCtx, (txnParticipant && opCtx->inMultiDocumentTransaction())); uassertStatusOK(userAllowedWriteNS(opCtx, ns)); - const auto [disableDocumentValidation, fleCrudProcessed] = - getDocumentValidationFlags(opCtx, wholeOp.getWriteCommandRequestBase()); - - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); - - DisableSafeContentValidationIfTrue safeContentValidationDisabler( - opCtx, disableDocumentValidation, fleCrudProcessed); - + DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler( + opCtx, wholeOp.getWriteCommandRequestBase().getBypassDocumentValidation()); LastOpFixer lastOpFixer(opCtx, ns); bool containsRetry = false; @@ -1266,15 +1227,8 @@ WriteResult performDeletes(OperationContext* opCtx, (txnParticipant && opCtx->inMultiDocumentTransaction())); uassertStatusOK(userAllowedWriteNS(opCtx, ns)); - const auto [disableDocumentValidation, fleCrudProcessed] = - getDocumentValidationFlags(opCtx, wholeOp.getWriteCommandRequestBase()); - - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); - - DisableSafeContentValidationIfTrue safeContentValidationDisabler( - opCtx, disableDocumentValidation, fleCrudProcessed); - + DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler( + opCtx, wholeOp.getWriteCommandRequestBase().getBypassDocumentValidation()); LastOpFixer lastOpFixer(opCtx, ns); bool containsRetry = false; diff --git a/src/mongo/db/ops/write_ops_exec.h b/src/mongo/db/ops/write_ops_exec.h index 3550a51c1ce..548a3034713 100644 --- a/src/mongo/db/ops/write_ops_exec.h +++ b/src/mongo/db/ops/write_ops_exec.h @@ -64,9 +64,6 @@ struct WriteResult { bool canContinue = true; }; -bool getFleCrudProcessed(OperationContext* opCtx, - const boost::optional<EncryptionInformation>& encryptionInfo); - /** * Performs a batch of inserts, updates, or deletes. * diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index 16e39fc4832..6b14f635095 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -99,7 +99,6 @@ env.Library( 'expression_context.cpp', 'expression_function.cpp', 'expression_js_emit.cpp', - 'expression_parser.idl', 'expression_test_api_version.cpp', 'expression_trigonometric.cpp', 'javascript_execution.cpp', @@ -107,7 +106,6 @@ env.Library( 'variables.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/crypto/fle_crypto', '$BUILD_DIR/mongo/db/bson/dotted_path_support', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', '$BUILD_DIR/mongo/db/exec/document_value/document_value', @@ -130,7 +128,6 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/mongohasher', '$BUILD_DIR/mongo/db/vector_clock', - '$BUILD_DIR/mongo/idl/idl_parser', ], ) diff --git a/src/mongo/db/pipeline/abt/agg_expression_visitor.cpp b/src/mongo/db/pipeline/abt/agg_expression_visitor.cpp index 55b44e4d37b..3d0c8453a47 100644 --- a/src/mongo/db/pipeline/abt/agg_expression_visitor.cpp +++ b/src/mongo/db/pipeline/abt/agg_expression_visitor.cpp @@ -302,10 +302,6 @@ public: unsupportedExpression(expr->getOpName()); } - void visit(const ExpressionInternalFLEEqual* expr) override final { - unsupportedExpression(expr->getOpName()); - } - void visit(const ExpressionMap* expr) override final { unsupportedExpression("$map"); } diff --git a/src/mongo/db/pipeline/dependencies.cpp b/src/mongo/db/pipeline/dependencies.cpp index d2a5563c7c7..8b60a31637c 100644 --- a/src/mongo/db/pipeline/dependencies.cpp +++ b/src/mongo/db/pipeline/dependencies.cpp @@ -37,13 +37,6 @@ namespace mongo { -std::list<std::string> DepsTracker::sortedFields() const { - // Use a special comparator to put parent fieldpaths before their children. - std::list<std::string> sortedFields(fields.begin(), fields.end()); - sortedFields.sort(PathPrefixComparator()); - return sortedFields; -} - BSONObj DepsTracker::toProjectionWithoutMetadata( TruncateToRootLevel truncationBehavior /*= TruncateToRootLevel::no*/) const { BSONObjBuilder bb; @@ -59,21 +52,17 @@ BSONObj DepsTracker::toProjectionWithoutMetadata( return bb.obj(); } - // Go through dependency fieldpaths to find the minimal set of projections that cover the - // dependencies. For example, the dependencies ["a.b", "a.b.c.g", "c", "c.d", "f"] would be - // minimally covered by the projection {"a.b": 1, "c": 1, "f": 1}. The key operation here is - // folding dependencies into ancestor dependencies, wherever possible. This is assisted by a - // special sort in DepsTracker::sortedFields that treats '.' as the first char and thus places - // parent paths directly before their children. bool idSpecified = false; std::string last; - for (const auto& field : sortedFields()) { + for (const auto& field : fields) { if (str::startsWith(field, "_id") && (field.size() == 3 || field[3] == '.')) { idSpecified = true; } if (!last.empty() && str::startsWith(field, last)) { - // We are including a parent of this field, so we can skip this field. + // we are including a parent of *it so we don't need to include this field + // explicitly. This logic relies on on set iterators going in lexicographic order so + // that a string is always directly before of all fields it prefixes. continue; } @@ -107,36 +96,4 @@ void DepsTracker::setNeedsMetadata(DocumentMetadataFields::MetaType type, bool r invariant(required || !_metadataDeps[type]); _metadataDeps[type] = required; } - -// Returns true if the lhs value should sort before the rhs, false otherwise. -bool PathPrefixComparator::operator()(const std::string& lhs, const std::string& rhs) const { - constexpr char dot = '.'; - - for (size_t pos = 0, len = std::min(lhs.size(), rhs.size()); pos < len; ++pos) { - // Below, we explicitly choose unsigned char because the usual const char& returned by - // operator[] is actually signed on x86 and will incorrectly order unicode characters. - unsigned char lchar = lhs[pos], rchar = rhs[pos]; - if (lchar == rchar) { - continue; - } - - // Consider the path delimiter '.' as being less than all other characters, so that - // paths sort directly before any paths they prefix and directly after any paths - // which prefix them. - if (lchar == dot) { - return true; - } else if (rchar == dot) { - return false; - } - - // Otherwise, default to normal character comparison. - return lchar < rchar; - } - - // If we get here, then we have reached the end of lhs and/or rhs and all of their path - // segments up to this point match. If lhs is shorter than rhs, then lhs prefixes rhs - // and should sort before it. - return lhs.size() < rhs.size(); -} - } // namespace mongo diff --git a/src/mongo/db/pipeline/dependencies.h b/src/mongo/db/pipeline/dependencies.h index 3c892de8181..bda3bf9b243 100644 --- a/src/mongo/db/pipeline/dependencies.h +++ b/src/mongo/db/pipeline/dependencies.h @@ -184,11 +184,6 @@ struct DepsTracker { } } - /** - * Return fieldpaths ordered such that a parent is immediately before its children. - */ - std::list<std::string> sortedFields() const; - std::set<std::string> fields; // Names of needed fields in dotted notation. std::set<Variables::Id> vars; // IDs of referenced variables. bool needWholeDocument = false; // If true, ignore 'fields'; the whole document is needed. @@ -206,13 +201,4 @@ private: // dependency analysis. QueryMetadataBitSet _metadataDeps; }; - - -/** Custom comparator that orders fieldpath strings by path prefix first, then by field. - * This ensures that a parent field is ordered directly before its children. - */ -struct PathPrefixComparator { - /* Returns true if the lhs value should sort before the rhs, false otherwise. */ - bool operator()(const std::string& lhs, const std::string& rhs) const; -}; } // namespace mongo diff --git a/src/mongo/db/pipeline/dependencies_test.cpp b/src/mongo/db/pipeline/dependencies_test.cpp index 938130b91bd..f366ad3ce1d 100644 --- a/src/mongo/db/pipeline/dependencies_test.cpp +++ b/src/mongo/db/pipeline/dependencies_test.cpp @@ -162,13 +162,6 @@ TEST(DependenciesToProjectionTest, ShouldIncludeFieldEvenIfSuffixOfAnotherFieldW BSON("a" << 1 << "ab" << 1 << "_id" << 0)); } -TEST(DependenciesToProjectionTest, ExcludeIndirectDescendants) { - const char* array[] = {"a.b", "_id", "a.b.c.d.e"}; - DepsTracker deps; - deps.fields = arrayToSet(array); - ASSERT_BSONOBJ_EQ(deps.toProjectionWithoutMetadata(), BSON("_id" << 1 << "a.b" << 1)); -} - TEST(DependenciesToProjectionTest, ShouldIncludeIdIfNeeded) { const char* array[] = {"a", "_id"}; DepsTracker deps; @@ -206,27 +199,6 @@ TEST(DependenciesToProjectionTest, ShouldIncludeFieldPrefixedByIdWhenIdSubfieldI BSON("_id.a" << 1 << "_id_a" << 1 << "a" << 1)); } -// SERVER-66418 -TEST(DependenciesToProjectionTest, ChildCoveredByParentWithSpecialChars) { - // without "_id" - { - // This is an important test case because '-' is one of the few chars before '.' in utf-8. - const char* array[] = {"a", "a-b", "a.b"}; - DepsTracker deps; - deps.fields = arrayToSet(array); - ASSERT_BSONOBJ_EQ(deps.toProjectionWithoutMetadata(), - BSON("a" << 1 << "a-b" << 1 << "_id" << 0)); - } - // with "_id" - { - const char* array[] = {"_id", "a", "a-b", "a.b"}; - DepsTracker deps; - deps.fields = arrayToSet(array); - ASSERT_BSONOBJ_EQ(deps.toProjectionWithoutMetadata(), - BSON("_id" << 1 << "a" << 1 << "a-b" << 1)); - } -} - TEST(DependenciesToProjectionTest, ShouldOutputEmptyObjectIfEntireDocumentNeeded) { const char* array[] = {"a"}; // fields ignored with needWholeDocument DepsTracker deps; @@ -287,56 +259,5 @@ TEST(DependenciesToProjectionTest, ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kTextScore]); } -TEST(DependenciesToProjectionTest, SortFieldPaths) { - const char* array[] = {"", - "A", - "_id", - "a", - "a.b", - "a.b.c", - "a.c", - // '-' char in utf-8 comes before '.' but our special fieldpath sort - // puts '.' first so that children directly follow their parents. - "a-b", - "a-b.ear", - "a-bear", - "a-bear.", - "a🌲", - "b", - "b.a" - "b.aa" - "b.🌲d"}; - DepsTracker deps; - deps.fields = arrayToSet(array); - // our custom sort will restore the ordering above - std::list<std::string> fieldPathSorted = deps.sortedFields(); - auto itr = fieldPathSorted.begin(); - for (unsigned long i = 0; i < fieldPathSorted.size(); i++) { - ASSERT_EQ(*itr, array[i]); - ++itr; - } -} - -TEST(DependenciesToProjectionTest, PathLessThan) { - auto lessThan = PathPrefixComparator(); - ASSERT_FALSE(lessThan("a", "a")); - ASSERT_TRUE(lessThan("a", "aa")); - ASSERT_TRUE(lessThan("a", "b")); - ASSERT_TRUE(lessThan("", "a")); - ASSERT_TRUE(lessThan("Aa", "aa")); - ASSERT_TRUE(lessThan("a.b", "ab")); - ASSERT_TRUE(lessThan("a.b", "a-b")); // SERVER-66418 - ASSERT_TRUE(lessThan("a.b", "a b")); // SERVER-66418 - // verify the difference from the standard sort - ASSERT_TRUE(std::string("a.b") > std::string("a-b")); - ASSERT_TRUE(std::string("a.b") > std::string("a b")); - // test unicode behavior - ASSERT_TRUE(lessThan("a.b", "a🌲")); - ASSERT_TRUE(lessThan("a.b", "a🌲b")); - ASSERT_TRUE(lessThan("🌲", "🌳")); // U+1F332 < U+1F333 - ASSERT_TRUE(lessThan("🌲", "🌲.b")); - ASSERT_FALSE(lessThan("🌲.b", "🌲")); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_densify.cpp b/src/mongo/db/pipeline/document_source_densify.cpp index 740ef193431..0325dbaa775 100644 --- a/src/mongo/db/pipeline/document_source_densify.cpp +++ b/src/mongo/db/pipeline/document_source_densify.cpp @@ -56,10 +56,6 @@ RangeStatement RangeStatement::parse(RangeSpec spec) { optional<TimeUnit> unit = [&]() { if (auto unit = spec.getUnit()) { - uassert(6586400, - "The step parameter in a range statement must be a whole number when " - "densifying a date range", - step.integral64Bit()); return optional<TimeUnit>(parseTimeUnit(unit.get())); } else { return optional<TimeUnit>(boost::none); @@ -279,8 +275,8 @@ DocumentSourceInternalDensify::DocGenerator::DocGenerator(DensifyValue min, // Extra checks for date step + unit. tassert(5733501, "Unit must be specified with a date step", _range.getUnit()); tassert(5733505, - "Step must be a whole number for date densification", - _range.getStep().integral64Bit()); + "Step must be representable as an integer for date densification", + _range.getStep().integral()); } else { tassert(5733506, "Unit must not be specified with non-date values", !_range.getUnit()); } @@ -881,7 +877,7 @@ DensifyValue DensifyValue::increment(const RangeStatement& range) const { }, [&](Date_t date) { return DensifyValue(dateAdd( - date, range.getUnit().value(), range.getStep().coerceToLong(), timezone())); + date, range.getUnit().value(), range.getStep().getDouble(), timezone())); }}, _value); } @@ -895,7 +891,7 @@ DensifyValue DensifyValue::decrement(const RangeStatement& range) const { }, [&](Date_t date) { return DensifyValue(dateAdd( - date, range.getUnit().value(), -range.getStep().coerceToLong(), timezone())); + date, range.getUnit().value(), -range.getStep().getDouble(), timezone())); }}, _value); } @@ -910,7 +906,7 @@ bool DensifyValue::isOnStepRelativeTo(DensifyValue base, RangeStatement range) c }, [&](Date_t date) { auto unit = range.getUnit().value(); - long long step = range.getStep().coerceToLong(); + double step = range.getStep().getDouble(); auto baseDate = base.getDate(); // Months, quarters and years have variable lengths depending on leap days diff --git a/src/mongo/db/pipeline/document_source_densify_test.cpp b/src/mongo/db/pipeline/document_source_densify_test.cpp index c1a0218e9eb..79b16303eb9 100644 --- a/src/mongo/db/pipeline/document_source_densify_test.cpp +++ b/src/mongo/db/pipeline/document_source_densify_test.cpp @@ -374,7 +374,7 @@ DEATH_TEST(DensifyGeneratorTest, DateMinMustBeLessThanMax, "lower or equal to") 5733502); } -DEATH_TEST(DensifyGeneratorTest, DateStepMustBeInt, "whole number") { +DEATH_TEST(DensifyGeneratorTest, DateStepMustBeInt, "integer") { size_t counter = 0; ASSERT_THROWS_CODE(GenClass(makeDate("2021-01-01T00:00:00.000Z"), RangeStatement(Value(1.5), diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index bf0915deddd..cc3728c024f 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -39,9 +39,6 @@ #include <utility> #include <vector> -#include "mongo/bson/bsonmisc.h" -#include "mongo/bson/bsontypes.h" -#include "mongo/crypto/fle_crypto.h" #include "mongo/db/bson/dotted_path_support.h" #include "mongo/db/commands/feature_compatibility_version_documentation.h" #include "mongo/db/exec/document_value/document.h" @@ -49,7 +46,6 @@ #include "mongo/db/hasher.h" #include "mongo/db/jsobj.h" #include "mongo/db/pipeline/expression_context.h" -#include "mongo/db/pipeline/expression_parser_gen.h" #include "mongo/db/pipeline/variable_validation.h" #include "mongo/db/query/datetime/date_time_support.h" #include "mongo/db/query/sort_pattern.h" @@ -3746,123 +3742,6 @@ const char* ExpressionLog10::getOpName() const { return "$log10"; } -/* ----------------------- ExpressionInternalFLEEqual ---------------------------- */ -constexpr auto kInternalFleEq = "$_internalFleEq"_sd; - -ExpressionInternalFLEEqual::ExpressionInternalFLEEqual(ExpressionContext* const expCtx, - boost::intrusive_ptr<Expression> field, - ConstDataRange serverToken, - int64_t contentionFactor, - ConstDataRange edcToken) - : Expression(expCtx, {std::move(field)}), - _serverToken(PrfBlockfromCDR(serverToken)), - _edcToken(PrfBlockfromCDR(edcToken)), - _contentionFactor(contentionFactor) { - expCtx->sbeCompatible = false; - - auto tokens = - EDCServerCollection::generateEDCTokens(ConstDataRange(_edcToken), _contentionFactor); - - for (auto& token : tokens) { - _cachedEDCTokens.insert(std::move(token.data)); - } -} - -void ExpressionInternalFLEEqual::_doAddDependencies(DepsTracker* deps) const { - for (auto&& operand : _children) { - operand->addDependencies(deps); - } -} - -REGISTER_EXPRESSION_WITH_MIN_VERSION(_internalFleEq, - ExpressionInternalFLEEqual::parse, - AllowedWithApiStrict::kAlways, - AllowedWithClientType::kAny, - multiversion::FeatureCompatibilityVersion::kVersion_6_0); - -intrusive_ptr<Expression> ExpressionInternalFLEEqual::parse(ExpressionContext* const expCtx, - BSONElement expr, - const VariablesParseState& vps) { - - IDLParserErrorContext ctx(kInternalFleEq); - auto fleEq = InternalFleEqStruct::parse(ctx, expr.Obj()); - - auto fieldExpr = Expression::parseOperand(expCtx, fleEq.getField().getElement(), vps); - - auto serverTokenPair = fromEncryptedConstDataRange(fleEq.getServerEncryptionToken()); - - uassert(6672405, - "Invalid server token", - serverTokenPair.first == EncryptedBinDataType::kFLE2TransientRaw && - serverTokenPair.second.length() == sizeof(PrfBlock)); - - auto edcTokenPair = fromEncryptedConstDataRange(fleEq.getEdcDerivedToken()); - - uassert(6672406, - "Invalid edc token", - edcTokenPair.first == EncryptedBinDataType::kFLE2TransientRaw && - edcTokenPair.second.length() == sizeof(PrfBlock)); - - - auto cf = fleEq.getMaxCounter(); - uassert(6672408, "Contention factor must be between 0 and 10000", cf >= 0 && cf < 10000); - - return new ExpressionInternalFLEEqual(expCtx, - std::move(fieldExpr), - serverTokenPair.second, - fleEq.getMaxCounter(), - edcTokenPair.second); -} - -Value toValue(const std::array<std::uint8_t, 32>& buf) { - auto vec = toEncryptedVector(EncryptedBinDataType::kFLE2TransientRaw, buf); - return Value(BSONBinData(vec.data(), vec.size(), BinDataType::Encrypt)); -} - -Value ExpressionInternalFLEEqual::serialize(bool explain) const { - return Value(Document{{kInternalFleEq, - Document{{"field", _children[0]->serialize(explain)}, - {"edc", toValue(_edcToken)}, - {"counter", Value(static_cast<long long>(_contentionFactor))}, - {"server", toValue(_serverToken)}}}}); -} - -Value ExpressionInternalFLEEqual::evaluate(const Document& root, Variables* variables) const { - // Inputs - // 1. Value for FLE2IndexedEqualityEncryptedValue field - - Value fieldValue = _children[0]->evaluate(root, variables); - - if (fieldValue.nullish()) { - return Value(BSONNULL); - } - - if (fieldValue.getType() != BinData) { - return Value(false); - } - - auto fieldValuePair = fromEncryptedBinData(fieldValue); - - uassert(6672407, - "Invalid encrypted indexed field", - fieldValuePair.first == EncryptedBinDataType::kFLE2EqualityIndexedValue); - - // Value matches if - // 1. Decrypt field is successful - // 2. EDC_u Token is in GenTokens(EDC Token, ContentionFactor) - // - auto swIndexed = - EDCServerCollection::decryptAndParse(ConstDataRange(_serverToken), fieldValuePair.second); - uassertStatusOK(swIndexed); - auto indexed = swIndexed.getValue(); - - return Value(_cachedEDCTokens.count(indexed.edc.data) == 1); -} - -const char* ExpressionInternalFLEEqual::getOpName() const { - return kInternalFleEq.rawData(); -} - /* ------------------------ ExpressionNary ----------------------------- */ /** diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h index 9cce6d0b1e2..fa59e89ecf2 100644 --- a/src/mongo/db/pipeline/expression.h +++ b/src/mongo/db/pipeline/expression.h @@ -29,7 +29,6 @@ #pragma once -#include "mongo/base/data_range.h" #include "mongo/platform/basic.h" #include <algorithm> @@ -2198,38 +2197,6 @@ public: } }; -class ExpressionInternalFLEEqual final : public Expression { -public: - ExpressionInternalFLEEqual(ExpressionContext* expCtx, - boost::intrusive_ptr<Expression> field, - ConstDataRange serverToken, - int64_t contentionFactor, - ConstDataRange edcToken); - Value serialize(bool explain) const final; - - Value evaluate(const Document& root, Variables* variables) const final; - const char* getOpName() const; - - static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, - BSONElement expr, - const VariablesParseState& vps); - void _doAddDependencies(DepsTracker* deps) const final; - - void acceptVisitor(ExpressionMutableVisitor* visitor) final { - return visitor->visit(this); - } - - void acceptVisitor(ExpressionConstVisitor* visitor) const final { - return visitor->visit(this); - } - -private: - std::array<std::uint8_t, 32> _serverToken; - std::array<std::uint8_t, 32> _edcToken; - int64_t _contentionFactor; - stdx::unordered_set<std::array<std::uint8_t, 32>> _cachedEDCTokens; -}; - class ExpressionMap final : public Expression { public: ExpressionMap( diff --git a/src/mongo/db/pipeline/expression_parser.idl b/src/mongo/db/pipeline/expression_parser.idl deleted file mode 100644 index 9f1cde70856..00000000000 --- a/src/mongo/db/pipeline/expression_parser.idl +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (C) 2022-present MongoDB, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the Server Side Public License, version 1, -# as published by MongoDB, Inc. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Server Side Public License for more details. -# -# You should have received a copy of the Server Side Public License -# along with this program. If not, see -# <http://www.mongodb.com/licensing/server-side-public-license>. -# -# As a special exception, the copyright holders give permission to link the -# code of portions of this program with the OpenSSL library under certain -# conditions as described in each individual source file and distribute -# linked combinations including the program with the OpenSSL library. You -# must comply with the Server Side Public License in all respects for -# all of the code used other than as permitted herein. If you modify file(s) -# with this exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do so, -# delete this exception statement from your version. If you delete this -# exception statement from all source files in the program, then also delete -# it in the license file. - -global: - cpp_namespace: "mongo" - -imports: - - "mongo/idl/basic_types.idl" - -structs: - - InternalFleEqStruct: - description: "Struct for $_internalFleEq" - strict: true - fields: - field: - description: "Expression" - type: IDLAnyType - cpp_name: field - edc: - description: "EDCDerivedFromDataToken" - type: bindata_encrypt - cpp_name: edcDerivedToken - server: - description: "ServerDataEncryptionLevel1Token" - type: bindata_encrypt - cpp_name: serverEncryptionToken - counter: - description: "Queryable Encryption max counter" - type: long - cpp_name: maxCounter - - diff --git a/src/mongo/db/pipeline/expression_test.cpp b/src/mongo/db/pipeline/expression_test.cpp index fd6f1c3490e..63eb5cc884c 100644 --- a/src/mongo/db/pipeline/expression_test.cpp +++ b/src/mongo/db/pipeline/expression_test.cpp @@ -169,7 +169,6 @@ void parseAndVerifyResults( ASSERT_VALUE_EQ(expr->evaluate({}, &expCtx.variables), expected); } - /* ------------------------- ExpressionArrayToObject -------------------------- */ TEST(ExpressionArrayToObjectTest, KVFormatSimple) { @@ -3716,240 +3715,4 @@ TEST(ExpressionCondTest, ConstantCondShouldOptimizeWithNonConstantBranches) { ASSERT_BSONOBJ_BINARY_EQ(expectedResult, expressionToBson(optimizedExprCond)); } -TEST(ExpressionFLETest, BadInputs) { - - auto expCtx = ExpressionContextForTest(); - auto vps = expCtx.variablesParseState; - { - auto expr = fromjson("{$_internalFleEq: 12}"); - ASSERT_THROWS_CODE(ExpressionInternalFLEEqual::parse(&expCtx, expr.firstElement(), vps), - DBException, - 10065); - } -} - -// Test we return true if it matches -TEST(ExpressionFLETest, TestBinData) { - auto expCtx = ExpressionContextForTest(); - auto vps = expCtx.variablesParseState; - - { - auto expr = fromjson(R"({$_internalFleEq: { - field: { - "$binary": { - "base64": - "BxI0VngSNJh2EjQSNFZ4kBIQ0JE8aMUFkPk5sSTVqfdNNfjqUfQQ1Uoj0BBcthrWoe9wyU3cN6zmWaQBPJ97t0ZPbecnMsU736yXre6cBO4Zdt/wThtY+v5+7vFgNnWpgRP0e+vam6QPmLvbBrO0LdsvAPTGW4yqwnzCIXCoEg7QPGfbfAXKPDTNenBfRlawiblmTOhO/6ljKotWsMp22q/rpHrn9IEIeJmecwuuPIJ7EA+XYQ3hOKVccYf2ogoK73+8xD/Vul83Qvr84Q8afc4QUMVs8A==", - "subType": "6" - } - }, - server: { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - }, - counter: { - "$numberLong": "3" - }, - edc: { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - } } })"); - auto exprFle = ExpressionInternalFLEEqual::parse(&expCtx, expr.firstElement(), vps); - - ASSERT_VALUE_EQ(exprFle->evaluate({}, &expCtx.variables), Value(true)); - } - - // Negative: Use wrong server token - { - auto expr = fromjson(R"({$_internalFleEq: { - field: { - "$binary": { - "base64": - "BxI0VngSNJh2EjQSNFZ4kBIQ0JE8aMUFkPk5sSTVqfdNNfjqUfQQ1Uoj0BBcthrWoe9wyU3cN6zmWaQBPJ97t0ZPbecnMsU736yXre6cBO4Zdt/wThtY+v5+7vFgNnWpgRP0e+vam6QPmLvbBrO0LdsvAPTGW4yqwnzCIXCoEg7QPGfbfAXKPDTNenBfRlawiblmTOhO/6ljKotWsMp22q/rpHrn9IEIeJmecwuuPIJ7EA+XYQ3hOKVccYf2ogoK73+8xD/Vul83Qvr84Q8afc4QUMVs8A==", - "subType": "6" - } - }, - server: { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - }, - counter: { - "$numberLong": "3" - }, - edc: { - "$binary": { - "base64": "CEWSMQID7SFWYAUI3ZKSFKATKRYDQFNXXEOGAD5D4RSG", - "subType": "6" - } - } } })"); - auto exprFle = ExpressionInternalFLEEqual::parse(&expCtx, expr.firstElement(), vps); - - ASSERT_VALUE_EQ(exprFle->evaluate({}, &expCtx.variables), Value(false)); - } - - // Negative: Use wrong edc token - { - auto expr = fromjson(R"({$_internalFleEq: { - field: { - "$binary": { - "base64": - "BxI0VngSNJh2EjQSNFZ4kBIQ0JE8aMUFkPk5sSTVqfdNNfjqUfQQ1Uoj0BBcthrWoe9wyU3cN6zmWaQBPJ97t0ZPbecnMsU736yXre6cBO4Zdt/wThtY+v5+7vFgNnWpgRP0e+vam6QPmLvbBrO0LdsvAPTGW4yqwnzCIXCoEg7QPGfbfAXKPDTNenBfRlawiblmTOhO/6ljKotWsMp22q/rpHrn9IEIeJmecwuuPIJ7EA+XYQ3hOKVccYf2ogoK73+8xD/Vul83Qvr84Q8afc4QUMVs8A==", - "subType": "6" - } - }, - server: { - "$binary": { - "base64": "COUAC/ERLYAKKX6B0VZ1R3QODOQFFJQJD+XLGIPU4/PS", - "subType": "6" - } - }, - counter: { - "$numberLong": "3" - }, - edc: { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - } } })"); - auto exprFle = ExpressionInternalFLEEqual::parse(&expCtx, expr.firstElement(), vps); - - ASSERT_THROWS_CODE( - exprFle->evaluate({}, &expCtx.variables), DBException, ErrorCodes::Overflow); - } -} - -TEST(ExpressionFLETest, TestBinData_ContentionFactor) { - auto expCtx = ExpressionContextForTest(); - auto vps = expCtx.variablesParseState; - - // Use the wrong contention factor - 0 - { - auto expr = fromjson(R"({$_internalFleEq: { - field: { - "$binary": { - "base64": - "BxI0VngSNJh2EjQSNFZ4kBIQ5+Wa5+SZafJeRUDGdLNx+i2ADDkyV2qA90Xcve7FqltoDm1PllSSgUS4fYtw3XDjzoNZrFFg8LfG2wH0HYbLMswv681KJpmEw7+RXy4CcPVFgoRFt24N13p7jT+pqu2oQAHAoxYTy/TsiAyY4RnAMiXYGg3hWz4AO/WxHNSyq6B6kX5d7x/hrXvppsZDc2Pmhd+c5xmovlv5RPj7wnNld13kYcMluztjNswiCH05hM/kp2/P7kw30iVnbz0SZxn1FjjCug==", - "subType": "6" - } - }, - server: { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - }, - counter: { - "$numberLong": "0" - }, - edc: { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - } } })"); - auto exprFle = ExpressionInternalFLEEqual::parse(&expCtx, expr.firstElement(), vps); - - ASSERT_VALUE_EQ(exprFle->evaluate({}, &expCtx.variables), Value(false)); - } - - // Use the right contention factor - 50 - { - auto expr = fromjson(R"({$_internalFleEq: { - field: { - "$binary": { - "base64": -"BxI0VngSNJh2EjQSNFZ4kBIQ5+Wa5+SZafJeRUDGdLNx+i2ADDkyV2qA90Xcve7FqltoDm1PllSSgUS4fYtw3XDjzoNZrFFg8LfG2wH0HYbLMswv681KJpmEw7+RXy4CcPVFgoRFt24N13p7jT+pqu2oQAHAoxYTy/TsiAyY4RnAMiXYGg3hWz4AO/WxHNSyq6B6kX5d7x/hrXvppsZDc2Pmhd+c5xmovlv5RPj7wnNld13kYcMluztjNswiCH05hM/kp2/P7kw30iVnbz0SZxn1FjjCug==", - "subType": "6" - } - }, - server: { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - }, - counter: { - "$numberLong": "50" - }, - edc: { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - } } })"); - auto exprFle = ExpressionInternalFLEEqual::parse(&expCtx, expr.firstElement(), vps); - - ASSERT_VALUE_EQ(exprFle->evaluate({}, &expCtx.variables), Value(true)); - } -} - -TEST(ExpressionFLETest, TestBinData_RoundTrip) { - auto expCtx = ExpressionContextForTest(); - auto vps = expCtx.variablesParseState; - - auto expr = fromjson(R"({$_internalFleEq: { - field: { - "$binary": { - "base64": - "BxI0VngSNJh2EjQSNFZ4kBIQ0JE8aMUFkPk5sSTVqfdNNfjqUfQQ1Uoj0BBcthrWoe9wyU3cN6zmWaQBPJ97t0ZPbecnMsU736yXre6cBO4Zdt/wThtY+v5+7vFgNnWpgRP0e+vam6QPmLvbBrO0LdsvAPTGW4yqwnzCIXCoEg7QPGfbfAXKPDTNenBfRlawiblmTOhO/6ljKotWsMp22q/rpHrn9IEIeJmecwuuPIJ7EA+XYQ3hOKVccYf2ogoK73+8xD/Vul83Qvr84Q8afc4QUMVs8A==", - "subType": "6" - } - }, - server: { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - }, - counter: { - "$numberLong": "3" - }, - edc: { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - } } })"); - auto exprFle = ExpressionInternalFLEEqual::parse(&expCtx, expr.firstElement(), vps); - - ASSERT_VALUE_EQ(exprFle->evaluate({}, &expCtx.variables), Value(true)); - - // Verify it round trips - auto value = exprFle->serialize(false); - - auto roundTripExpr = fromjson(R"({$_internalFleEq: { - field: { - "$const" : { "$binary": { - "base64": - "BxI0VngSNJh2EjQSNFZ4kBIQ0JE8aMUFkPk5sSTVqfdNNfjqUfQQ1Uoj0BBcthrWoe9wyU3cN6zmWaQBPJ97t0ZPbecnMsU736yXre6cBO4Zdt/wThtY+v5+7vFgNnWpgRP0e+vam6QPmLvbBrO0LdsvAPTGW4yqwnzCIXCoEg7QPGfbfAXKPDTNenBfRlawiblmTOhO/6ljKotWsMp22q/rpHrn9IEIeJmecwuuPIJ7EA+XYQ3hOKVccYf2ogoK73+8xD/Vul83Qvr84Q8afc4QUMVs8A==", - "subType": "6" - }} - }, - edc: { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - }, - counter: { - "$numberLong": "3" - }, - server: { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - } })"); - - - ASSERT_BSONOBJ_EQ(value.getDocument().toBson(), roundTripExpr); -} - } // namespace ExpressionTests diff --git a/src/mongo/db/pipeline/expression_visitor.h b/src/mongo/db/pipeline/expression_visitor.h index 6b7c4fc4cdd..46ad3ee6295 100644 --- a/src/mongo/db/pipeline/expression_visitor.h +++ b/src/mongo/db/pipeline/expression_visitor.h @@ -153,7 +153,6 @@ class ExpressionHyperbolicSine; class ExpressionInternalFindSlice; class ExpressionInternalFindPositional; class ExpressionInternalFindElemMatch; -class ExpressionInternalFLEEqual; class ExpressionInternalJsEmit; class ExpressionFunction; class ExpressionDegreesToRadians; @@ -246,7 +245,6 @@ public: virtual void visit(expression_walker::MaybeConstPtr<IsConst, ExpressionLn>) = 0; virtual void visit(expression_walker::MaybeConstPtr<IsConst, ExpressionLog>) = 0; virtual void visit(expression_walker::MaybeConstPtr<IsConst, ExpressionLog10>) = 0; - virtual void visit(expression_walker::MaybeConstPtr<IsConst, ExpressionInternalFLEEqual>) = 0; virtual void visit(expression_walker::MaybeConstPtr<IsConst, ExpressionMap>) = 0; virtual void visit(expression_walker::MaybeConstPtr<IsConst, ExpressionMeta>) = 0; virtual void visit(expression_walker::MaybeConstPtr<IsConst, ExpressionMod>) = 0; @@ -426,7 +424,6 @@ struct SelectiveConstExpressionVisitorBase : public ExpressionConstVisitor { void visit(const ExpressionLn*) override {} void visit(const ExpressionLog*) override {} void visit(const ExpressionLog10*) override {} - void visit(const ExpressionInternalFLEEqual*) override {} void visit(const ExpressionMap*) override {} void visit(const ExpressionMeta*) override {} void visit(const ExpressionMod*) override {} diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp index b21e8635c51..85c3e82839a 100644 --- a/src/mongo/db/pipeline/pipeline_d.cpp +++ b/src/mongo/db/pipeline/pipeline_d.cpp @@ -1451,9 +1451,7 @@ PipelineD::buildInnerQueryExecutorGeneric(const MultipleCollectionAccessor& coll // This produces {$const: maxBucketSpanSeconds} make_intrusive<ExpressionConstant>( expCtx.get(), - Value{static_cast<long long>( - unpack->getBucketMaxSpanSeconds()) * - 1000}))), + Value{unpack->getBucketMaxSpanSeconds() * 1000}))), expCtx); pipeline->_sources.insert( unpackIter, diff --git a/src/mongo/db/query/fle/server_rewrite.cpp b/src/mongo/db/query/fle/server_rewrite.cpp index 1660c368093..185a979ec3d 100644 --- a/src/mongo/db/query/fle/server_rewrite.cpp +++ b/src/mongo/db/query/fle/server_rewrite.cpp @@ -27,13 +27,11 @@ * it in the license file. */ -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery #include "mongo/db/query/fle/server_rewrite.h" #include <memory> -#include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/bsontypes.h" @@ -50,11 +48,9 @@ #include "mongo/db/pipeline/expression.h" #include "mongo/db/query/collation/collator_factory_interface.h" #include "mongo/db/service_context.h" -#include "mongo/logv2/log.h" #include "mongo/s/grid.h" #include "mongo/s/transaction_router_resource_yielder.h" #include "mongo/util/assert_util.h" -#include "mongo/util/intrusive_counter.h" namespace mongo::fle { @@ -72,56 +68,6 @@ std::unique_ptr<CollatorInterface> collatorFromBSON(OperationContext* opCtx, } namespace { -template <typename PayloadT> -boost::intrusive_ptr<ExpressionInternalFLEEqual> generateFleEqualMatch(StringData path, - const PayloadT& ffp, - ExpressionContext* expCtx) { - // Generate { $_internalFleEq: { field: "$field_name", server: f_3, counter: cm, edc: k_EDC] } - auto tokens = ParsedFindPayload(ffp); - - uassert(6672401, - "Missing required field server encryption token in find payload", - tokens.serverToken.has_value()); - - return make_intrusive<ExpressionInternalFLEEqual>( - expCtx, - ExpressionFieldPath::createPathFromString( - expCtx, path.toString(), expCtx->variablesParseState), - tokens.serverToken.get().data, - tokens.maxCounter.value_or(0LL), - tokens.edcToken.data); -} - - -template <typename PayloadT> -std::unique_ptr<ExpressionInternalFLEEqual> generateFleEqualMatchUnique(StringData path, - const PayloadT& ffp, - ExpressionContext* expCtx) { - // Generate { $_internalFleEq: { field: "$field_name", server: f_3, counter: cm, edc: k_EDC] } - auto tokens = ParsedFindPayload(ffp); - - uassert(6672419, - "Missing required field server encryption token in find payload", - tokens.serverToken.has_value()); - - return std::make_unique<ExpressionInternalFLEEqual>( - expCtx, - ExpressionFieldPath::createPathFromString( - expCtx, path.toString(), expCtx->variablesParseState), - tokens.serverToken.get().data, - tokens.maxCounter.value_or(0LL), - tokens.edcToken.data); -} - -std::unique_ptr<MatchExpression> generateFleEqualMatchAndExpr(StringData path, - const BSONElement ffp, - ExpressionContext* expCtx) { - auto fleEqualMatch = generateFleEqualMatch(path, ffp, expCtx); - - return std::make_unique<ExprMatchExpression>(fleEqualMatch, expCtx); -} - - /** * This section defines a mapping from DocumentSources to the dispatch function to appropriately * handle FLE rewriting for that stage. This should be kept in line with code on the client-side @@ -182,8 +128,7 @@ public: * The final output will look like * {$or: [{$in: [tag0, "$__safeContent__"]}, {$in: [tag1, "$__safeContent__"]}, ...]}. */ - std::unique_ptr<Expression> rewriteInToEncryptedField( - const Expression* leftExpr, + std::unique_ptr<Expression> rewriteComparisonsToEncryptedField( const std::vector<boost::intrusive_ptr<Expression>>& equalitiesList) { size_t numFFPs = 0; std::vector<boost::intrusive_ptr<Expression>> orListElems; @@ -195,122 +140,11 @@ public: continue; } + // ... rewrite the payload to a list of tags... numFFPs++; - } - } - - // Finally, construct an $or of all of the $ins. - if (numFFPs == 0) { - return nullptr; - } - - uassert( - 6334102, - "If any elements in an comparison expression are encrypted, then all elements should " - "be encrypted.", - numFFPs == equalitiesList.size()); - - auto leftFieldPath = dynamic_cast<const ExpressionFieldPath*>(leftExpr); - uassert(6672417, - "$in is only supported with Queryable Encryption when the first argument is a " - "field path", - leftFieldPath != nullptr); - - if (!queryRewriter->isForceHighCardinality()) { - try { - for (auto& equality : equalitiesList) { - // For each expression representing a FleFindPayload... - if (auto constChild = dynamic_cast<ExpressionConstant*>(equality.get())) { - // ... rewrite the payload to a list of tags... - auto tags = queryRewriter->rewritePayloadAsTags(constChild->getValue()); - for (auto&& tagElt : tags) { - // ... and for each tag, construct expression {$in: [tag, - // "$__safeContent__"]}. - std::vector<boost::intrusive_ptr<Expression>> inVec{ - ExpressionConstant::create(queryRewriter->expCtx(), tagElt), - ExpressionFieldPath::createPathFromString( - queryRewriter->expCtx(), - kSafeContent, - queryRewriter->expCtx()->variablesParseState)}; - orListElems.push_back(make_intrusive<ExpressionIn>( - queryRewriter->expCtx(), std::move(inVec))); - } - } - } - - didRewrite = true; - - return std::make_unique<ExpressionOr>(queryRewriter->expCtx(), - std::move(orListElems)); - } catch (const ExceptionFor<ErrorCodes::FLEMaxTagLimitExceeded>& ex) { - LOGV2_DEBUG(6672403, - 2, - "FLE Max tag limit hit during aggregation $in rewrite", - "__error__"_attr = ex.what()); - - if (queryRewriter->getHighCardinalityMode() != - FLEQueryRewriter::HighCardinalityMode::kUseIfNeeded) { - throw; - } - - // fall through - } - } - - for (auto& equality : equalitiesList) { - if (auto constChild = dynamic_cast<ExpressionConstant*>(equality.get())) { - auto fleEqExpr = generateFleEqualMatch( - leftFieldPath->getFieldPathWithoutCurrentPrefix().fullPath(), - constChild->getValue(), - queryRewriter->expCtx()); - orListElems.push_back(fleEqExpr); - } - } - - didRewrite = true; - return std::make_unique<ExpressionOr>(queryRewriter->expCtx(), std::move(orListElems)); - } - - // Rewrite a [$eq : [$fieldpath, constant]] or [$eq: [constant, $fieldpath]] - // to _internalFleEq: {field: $fieldpath, edc: edcToken, counter: N, server: serverToken} - std::unique_ptr<Expression> rewriteComparisonsToEncryptedField( - const std::vector<boost::intrusive_ptr<Expression>>& equalitiesList) { - - auto leftConstant = dynamic_cast<ExpressionConstant*>(equalitiesList[0].get()); - auto rightConstant = dynamic_cast<ExpressionConstant*>(equalitiesList[1].get()); - - bool isLeftFFP = leftConstant && queryRewriter->isFleFindPayload(leftConstant->getValue()); - bool isRightFFP = - rightConstant && queryRewriter->isFleFindPayload(rightConstant->getValue()); - - uassert(6334100, - "Cannot compare two encrypted constants to each other", - !(isLeftFFP && isRightFFP)); - - // No FLE Find Payload - if (!isLeftFFP && !isRightFFP) { - return nullptr; - } - - auto leftFieldPath = dynamic_cast<ExpressionFieldPath*>(equalitiesList[0].get()); - auto rightFieldPath = dynamic_cast<ExpressionFieldPath*>(equalitiesList[1].get()); - - uassert( - 6672413, - "Queryable Encryption only supports comparisons between a field path and a constant", - leftFieldPath || rightFieldPath); - - auto fieldPath = leftFieldPath ? leftFieldPath : rightFieldPath; - auto constChild = isLeftFFP ? leftConstant : rightConstant; - - if (!queryRewriter->isForceHighCardinality()) { - try { - std::vector<boost::intrusive_ptr<Expression>> orListElems; - auto tags = queryRewriter->rewritePayloadAsTags(constChild->getValue()); for (auto&& tagElt : tags) { - // ... and for each tag, construct expression {$in: [tag, - // "$__safeContent__"]}. + // ... and for each tag, construct expression {$in: [tag, "$__safeContent__"]}. std::vector<boost::intrusive_ptr<Expression>> inVec{ ExpressionConstant::create(queryRewriter->expCtx(), tagElt), ExpressionFieldPath::createPathFromString( @@ -320,33 +154,21 @@ public: orListElems.push_back( make_intrusive<ExpressionIn>(queryRewriter->expCtx(), std::move(inVec))); } - - didRewrite = true; - return std::make_unique<ExpressionOr>(queryRewriter->expCtx(), - std::move(orListElems)); - - } catch (const ExceptionFor<ErrorCodes::FLEMaxTagLimitExceeded>& ex) { - LOGV2_DEBUG(6672409, - 2, - "FLE Max tag limit hit during query $in rewrite", - "__error__"_attr = ex.what()); - - if (queryRewriter->getHighCardinalityMode() != - FLEQueryRewriter::HighCardinalityMode::kUseIfNeeded) { - throw; - } - - // fall through } } - auto fleEqExpr = - generateFleEqualMatchUnique(fieldPath->getFieldPathWithoutCurrentPrefix().fullPath(), - constChild->getValue(), - queryRewriter->expCtx()); + // Finally, construct an $or of all of the $ins. + if (numFFPs == 0) { + return nullptr; + } + uassert( + 6334102, + "If any elements in an comparison expression are encrypted, then all elements should " + "be encrypted.", + numFFPs == equalitiesList.size()); didRewrite = true; - return fleEqExpr; + return std::make_unique<ExpressionOr>(queryRewriter->expCtx(), std::move(orListElems)); } std::unique_ptr<Expression> postVisit(Expression* exp) { @@ -355,28 +177,30 @@ public: // ignored when rewrites are done; there is no extra information in that child that // doesn't exist in the FFPs in the $in list. if (auto inList = dynamic_cast<ExpressionArray*>(inExpr->getOperandList()[1].get())) { - return rewriteInToEncryptedField(inExpr->getOperandList()[0].get(), - inList->getChildren()); + return rewriteComparisonsToEncryptedField(inList->getChildren()); } } else if (auto eqExpr = dynamic_cast<ExpressionCompare*>(exp); eqExpr && (eqExpr->getOp() == ExpressionCompare::EQ || eqExpr->getOp() == ExpressionCompare::NE)) { // Rewrite an $eq comparing an encrypted field and an encrypted constant to an $or. - auto newExpr = rewriteComparisonsToEncryptedField(eqExpr->getChildren()); + // Either child may be the constant, so try rewriting both. + auto or0 = rewriteComparisonsToEncryptedField({eqExpr->getChildren()[0]}); + auto or1 = rewriteComparisonsToEncryptedField({eqExpr->getChildren()[1]}); + uassert(6334100, "Cannot compare two encrypted constants to each other", !or0 || !or1); // Neither child is an encrypted constant, and no rewriting needs to be done. - if (!newExpr) { + if (!or0 && !or1) { return nullptr; } // Exactly one child was an encrypted constant. The other child can be ignored; there is // no extra information in that child that doesn't exist in the FFP. if (eqExpr->getOp() == ExpressionCompare::NE) { - std::vector<boost::intrusive_ptr<Expression>> notChild{newExpr.release()}; + std::vector<boost::intrusive_ptr<Expression>> notChild{(or0 ? or0 : or1).release()}; return std::make_unique<ExpressionNot>(queryRewriter->expCtx(), std::move(notChild)); } - return newExpr; + return std::move(or0 ? or0 : or1); } return nullptr; @@ -389,14 +213,11 @@ public: BSONObj rewriteEncryptedFilter(const FLEStateCollectionReader& escReader, const FLEStateCollectionReader& eccReader, boost::intrusive_ptr<ExpressionContext> expCtx, - BSONObj filter, - HighCardinalityModeAllowed mode) { - + BSONObj filter) { if (auto rewritten = - FLEQueryRewriter(expCtx, escReader, eccReader, mode).rewriteMatchExpression(filter)) { + FLEQueryRewriter(expCtx, escReader, eccReader).rewriteMatchExpression(filter)) { return rewritten.get(); } - return filter; } @@ -452,18 +273,16 @@ public: FilterRewrite(boost::intrusive_ptr<ExpressionContext> expCtx, const NamespaceString& nss, const EncryptionInformation& encryptInfo, - const BSONObj toRewrite, - HighCardinalityModeAllowed mode) - : RewriteBase(expCtx, nss, encryptInfo), userFilter(toRewrite), _mode(mode) {} + const BSONObj toRewrite) + : RewriteBase(expCtx, nss, encryptInfo), userFilter(toRewrite) {} ~FilterRewrite(){}; void doRewrite(FLEStateCollectionReader& escReader, FLEStateCollectionReader& eccReader) final { - rewrittenFilter = rewriteEncryptedFilter(escReader, eccReader, expCtx, userFilter, _mode); + rewrittenFilter = rewriteEncryptedFilter(escReader, eccReader, expCtx, userFilter); } const BSONObj userFilter; BSONObj rewrittenFilter; - HighCardinalityModeAllowed _mode; }; // This helper executes the rewrite(s) inside a transaction. The transaction runs in a separate @@ -505,8 +324,7 @@ BSONObj rewriteEncryptedFilterInsideTxn(FLEQueryInterface* queryImpl, StringData db, const EncryptedFieldConfig& efc, boost::intrusive_ptr<ExpressionContext> expCtx, - BSONObj filter, - HighCardinalityModeAllowed mode) { + BSONObj filter) { auto makeCollectionReader = [&](FLEQueryInterface* queryImpl, const StringData& coll) { NamespaceString nss(db, coll); auto docCount = queryImpl->countDocuments(nss); @@ -514,8 +332,7 @@ BSONObj rewriteEncryptedFilterInsideTxn(FLEQueryInterface* queryImpl, }; auto escReader = makeCollectionReader(queryImpl, efc.getEscCollection().get()); auto eccReader = makeCollectionReader(queryImpl, efc.getEccCollection().get()); - - return rewriteEncryptedFilter(escReader, eccReader, expCtx, filter, mode); + return rewriteEncryptedFilter(escReader, eccReader, expCtx, filter); } BSONObj rewriteQuery(OperationContext* opCtx, @@ -523,9 +340,8 @@ BSONObj rewriteQuery(OperationContext* opCtx, const NamespaceString& nss, const EncryptionInformation& info, BSONObj filter, - GetTxnCallback getTransaction, - HighCardinalityModeAllowed mode) { - auto sharedBlock = std::make_shared<FilterRewrite>(expCtx, nss, info, filter, mode); + GetTxnCallback getTransaction) { + auto sharedBlock = std::make_shared<FilterRewrite>(expCtx, nss, info, filter); doFLERewriteInTxn(opCtx, sharedBlock, getTransaction); return sharedBlock->rewrittenFilter.getOwned(); } @@ -549,8 +365,7 @@ void processFindCommand(OperationContext* opCtx, nss, findCommand->getEncryptionInformation().get(), findCommand->getFilter().getOwned(), - getTransaction, - HighCardinalityModeAllowed::kAllow)); + getTransaction)); // The presence of encryptionInformation is a signal that this is a FLE request that requires // special processing. Once we've rewritten the query, it's no longer a "special" FLE query, but // a normal query that can be executed by the query system like any other, so remove @@ -574,8 +389,7 @@ void processCountCommand(OperationContext* opCtx, nss, countCommand->getEncryptionInformation().get(), countCommand->getQuery().getOwned(), - getTxn, - HighCardinalityModeAllowed::kAllow)); + getTxn)); // The presence of encryptionInformation is a signal that this is a FLE request that requires // special processing. Once we've rewritten the query, it's no longer a "special" FLE query, but // a normal query that can be executed by the query system like any other, so remove @@ -689,112 +503,59 @@ std::vector<Value> FLEQueryRewriter::rewritePayloadAsTags(Value fleFindPayload) return tagVec; } - -std::unique_ptr<MatchExpression> FLEQueryRewriter::rewriteEq(const EqualityMatchExpression* expr) { +std::unique_ptr<InMatchExpression> FLEQueryRewriter::rewriteEq( + const EqualityMatchExpression* expr) { auto ffp = expr->getData(); if (!isFleFindPayload(ffp)) { return nullptr; } - if (_mode != HighCardinalityMode::kForceAlways) { - try { - auto obj = rewritePayloadAsTags(ffp); - - auto tags = std::vector<BSONElement>(); - obj.elems(tags); - - auto inExpr = std::make_unique<InMatchExpression>(kSafeContent); - inExpr->setBackingBSON(std::move(obj)); - auto status = inExpr->setEqualities(std::move(tags)); - uassertStatusOK(status); - _rewroteLastExpression = true; - return inExpr; - } catch (const ExceptionFor<ErrorCodes::FLEMaxTagLimitExceeded>& ex) { - LOGV2_DEBUG(6672410, - 2, - "FLE Max tag limit hit during query $eq rewrite", - "__error__"_attr = ex.what()); - - if (_mode != HighCardinalityMode::kUseIfNeeded) { - throw; - } + auto obj = rewritePayloadAsTags(ffp); - // fall through - } - } + auto tags = std::vector<BSONElement>(); + obj.elems(tags); + + auto inExpr = std::make_unique<InMatchExpression>(kSafeContent); + inExpr->setBackingBSON(std::move(obj)); + auto status = inExpr->setEqualities(std::move(tags)); + uassertStatusOK(status); - auto exprMatch = generateFleEqualMatchAndExpr(expr->path(), ffp, _expCtx.get()); _rewroteLastExpression = true; - return exprMatch; + return inExpr; } -std::unique_ptr<MatchExpression> FLEQueryRewriter::rewriteIn(const InMatchExpression* expr) { +std::unique_ptr<InMatchExpression> FLEQueryRewriter::rewriteIn(const InMatchExpression* expr) { + auto backingBSONBuilder = BSONArrayBuilder(); size_t numFFPs = 0; for (auto& eq : expr->getEqualities()) { if (isFleFindPayload(eq)) { + auto obj = rewritePayloadAsTags(eq); ++numFFPs; + for (auto&& elt : obj) { + backingBSONBuilder.append(elt); + } } } - if (numFFPs == 0) { return nullptr; } - // All elements in an encrypted $in expression should be FFPs. uassert( 6329400, "If any elements in a $in expression are encrypted, then all elements should be encrypted.", numFFPs == expr->getEqualities().size()); - if (_mode != HighCardinalityMode::kForceAlways) { - - try { - auto backingBSONBuilder = BSONArrayBuilder(); - - for (auto& eq : expr->getEqualities()) { - auto obj = rewritePayloadAsTags(eq); - for (auto&& elt : obj) { - backingBSONBuilder.append(elt); - } - } - - auto backingBSON = backingBSONBuilder.arr(); - auto allTags = std::vector<BSONElement>(); - backingBSON.elems(allTags); - - auto inExpr = std::make_unique<InMatchExpression>(kSafeContent); - inExpr->setBackingBSON(std::move(backingBSON)); - auto status = inExpr->setEqualities(std::move(allTags)); - uassertStatusOK(status); - - _rewroteLastExpression = true; - return inExpr; - - } catch (const ExceptionFor<ErrorCodes::FLEMaxTagLimitExceeded>& ex) { - LOGV2_DEBUG(6672411, - 2, - "FLE Max tag limit hit during query $in rewrite", - "__error__"_attr = ex.what()); - - if (_mode != HighCardinalityMode::kUseIfNeeded) { - throw; - } - - // fall through - } - } - - std::vector<std::unique_ptr<MatchExpression>> matches; - matches.reserve(numFFPs); + auto backingBSON = backingBSONBuilder.arr(); + auto allTags = std::vector<BSONElement>(); + backingBSON.elems(allTags); - for (auto& eq : expr->getEqualities()) { - auto exprMatch = generateFleEqualMatchAndExpr(expr->path(), eq, _expCtx.get()); - matches.push_back(std::move(exprMatch)); - } + auto inExpr = std::make_unique<InMatchExpression>(kSafeContent); + inExpr->setBackingBSON(std::move(backingBSON)); + auto status = inExpr->setEqualities(std::move(allTags)); + uassertStatusOK(status); - auto orExpr = std::make_unique<OrMatchExpression>(std::move(matches)); _rewroteLastExpression = true; - return orExpr; + return inExpr; } } // namespace mongo::fle diff --git a/src/mongo/db/query/fle/server_rewrite.h b/src/mongo/db/query/fle/server_rewrite.h index bf02eeebd4e..c71d1f01392 100644 --- a/src/mongo/db/query/fle/server_rewrite.h +++ b/src/mongo/db/query/fle/server_rewrite.h @@ -31,7 +31,7 @@ #include <memory> -#include <boost/smart_ptr/intrusive_ptr.hpp> +#include "boost/smart_ptr/intrusive_ptr.hpp" #include "mongo/bson/bsonobj.h" #include "mongo/crypto/fle_crypto.h" @@ -47,14 +47,6 @@ class FLEQueryInterface; namespace fle { /** - * Low Selectivity rewrites use $expr which is not supported in all commands such as upserts. - */ -enum class HighCardinalityModeAllowed { - kAllow, - kDisallow, -}; - -/** * Make a collator object from its BSON representation. Useful when creating ExpressionContext * objects for parsing MatchExpressions as part of the server-side rewrite. */ @@ -70,8 +62,7 @@ BSONObj rewriteQuery(OperationContext* opCtx, const NamespaceString& nss, const EncryptionInformation& info, BSONObj filter, - GetTxnCallback getTransaction, - HighCardinalityModeAllowed mode); + GetTxnCallback getTransaction); /** * Process a find command with encryptionInformation in-place, rewriting the filter condition so @@ -109,13 +100,11 @@ std::unique_ptr<Pipeline, PipelineDeleter> processPipeline( * from inside an existing transaction using a FLEQueryInterface constructed from a * transaction client. */ -BSONObj rewriteEncryptedFilterInsideTxn( - FLEQueryInterface* queryImpl, - StringData db, - const EncryptedFieldConfig& efc, - boost::intrusive_ptr<ExpressionContext> expCtx, - BSONObj filter, - HighCardinalityModeAllowed mode = HighCardinalityModeAllowed::kDisallow); +BSONObj rewriteEncryptedFilterInsideTxn(FLEQueryInterface* queryImpl, + StringData db, + const EncryptedFieldConfig& efc, + boost::intrusive_ptr<ExpressionContext> expCtx, + BSONObj filter); /** * Class which handles rewriting filter MatchExpressions for FLE2. The functionality is encapsulated @@ -127,37 +116,14 @@ BSONObj rewriteEncryptedFilterInsideTxn( */ class FLEQueryRewriter { public: - enum class HighCardinalityMode { - // Always use high cardinality filters, used by tests - kForceAlways, - - // Use high cardinality mode if $in rewrites do not fit in the - // internalQueryFLERewriteMemoryLimit memory limit - kUseIfNeeded, - - // Do not rewrite into high cardinality filter, throw exceptions instead - // Some contexts like upsert do not support $expr - kDisallow, - }; - /** * Takes in references to collection readers for the ESC and ECC that are used during tag * computation. */ FLEQueryRewriter(boost::intrusive_ptr<ExpressionContext> expCtx, const FLEStateCollectionReader& escReader, - const FLEStateCollectionReader& eccReader, - HighCardinalityModeAllowed mode = HighCardinalityModeAllowed::kAllow) + const FLEStateCollectionReader& eccReader) : _expCtx(expCtx), _escReader(&escReader), _eccReader(&eccReader) { - - if (internalQueryFLEAlwaysUseHighCardinalityMode.load()) { - _mode = HighCardinalityMode::kForceAlways; - } - - if (mode == HighCardinalityModeAllowed::kDisallow) { - _mode = HighCardinalityMode::kDisallow; - } - // This isn't the "real" query so we don't want to increment Expression // counters here. _expCtx->stopExpressionCounters(); @@ -218,18 +184,6 @@ public: return _expCtx.get(); } - bool isForceHighCardinality() const { - return _mode == HighCardinalityMode::kForceAlways; - } - - void setForceHighCardinalityForTest() { - _mode = HighCardinalityMode::kForceAlways; - } - - HighCardinalityMode getHighCardinalityMode() const { - return _mode; - } - protected: // This constructor should only be used for mocks in testing. FLEQueryRewriter(boost::intrusive_ptr<ExpressionContext> expCtx) @@ -242,8 +196,8 @@ private: std::unique_ptr<MatchExpression> _rewrite(MatchExpression* me); virtual BSONObj rewritePayloadAsTags(BSONElement fleFindPayload) const; - std::unique_ptr<MatchExpression> rewriteEq(const EqualityMatchExpression* expr); - std::unique_ptr<MatchExpression> rewriteIn(const InMatchExpression* expr); + std::unique_ptr<InMatchExpression> rewriteEq(const EqualityMatchExpression* expr); + std::unique_ptr<InMatchExpression> rewriteIn(const InMatchExpression* expr); boost::intrusive_ptr<ExpressionContext> _expCtx; @@ -253,10 +207,7 @@ private: const FLEStateCollectionReader* _eccReader; // True if the last Expression or MatchExpression processed by this rewriter was rewritten. - bool _rewroteLastExpression = false; - - // Controls how query rewriter rewrites the query - HighCardinalityMode _mode{HighCardinalityMode::kUseIfNeeded}; + bool _rewroteLastExpression; }; diff --git a/src/mongo/db/query/fle/server_rewrite_test.cpp b/src/mongo/db/query/fle/server_rewrite_test.cpp index 034de8f0aa9..cb81656dcb6 100644 --- a/src/mongo/db/query/fle/server_rewrite_test.cpp +++ b/src/mongo/db/query/fle/server_rewrite_test.cpp @@ -31,9 +31,7 @@ #include <memory> #include "mongo/bson/bsonelement.h" -#include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobjbuilder.h" -#include "mongo/bson/bsontypes.h" #include "mongo/db/matcher/expression_leaf.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/fle/server_rewrite.h" @@ -44,19 +42,9 @@ namespace mongo { namespace { -class BasicMockFLEQueryRewriter : public fle::FLEQueryRewriter { +class MockFLEQueryRewriter : public fle::FLEQueryRewriter { public: - BasicMockFLEQueryRewriter() : fle::FLEQueryRewriter(new ExpressionContextForTest()) {} - - BSONObj rewriteMatchExpressionForTest(const BSONObj& obj) { - auto res = rewriteMatchExpression(obj); - return res ? res.get() : obj; - } -}; - -class MockFLEQueryRewriter : public BasicMockFLEQueryRewriter { -public: - MockFLEQueryRewriter() : _tags() {} + MockFLEQueryRewriter() : fle::FLEQueryRewriter(new ExpressionContextForTest()), _tags() {} bool isFleFindPayload(const BSONElement& fleFindPayload) const override { return _encryptedFields.find(fleFindPayload.fieldNameStringData()) != @@ -68,6 +56,11 @@ public: _tags[fieldvalue] = tags; } + BSONObj rewriteMatchExpressionForTest(const BSONObj& obj) { + auto res = rewriteMatchExpression(obj); + return res ? res.get() : obj; + } + private: BSONObj rewritePayloadAsTags(BSONElement fleFindPayload) const override { ASSERT(fleFindPayload.isNumber()); // Only accept numbers as mock FFPs. @@ -79,7 +72,6 @@ private: std::map<std::pair<StringData, int>, BSONObj> _tags; std::set<StringData> _encryptedFields; }; - class FLEServerRewriteTest : public unittest::Test { public: FLEServerRewriteTest() {} @@ -369,290 +361,5 @@ TEST_F(FLEServerRewriteTest, ComparisonToObjectIgnored) { } } -template <typename T> -std::vector<uint8_t> toEncryptedVector(EncryptedBinDataType dt, T t) { - BSONObj obj = t.toBSON(); - - std::vector<uint8_t> buf(obj.objsize() + 1); - buf[0] = static_cast<uint8_t>(dt); - - std::copy(obj.objdata(), obj.objdata() + obj.objsize(), buf.data() + 1); - - return buf; -} - -template <typename T> -void toEncryptedBinData(StringData field, EncryptedBinDataType dt, T t, BSONObjBuilder* builder) { - auto buf = toEncryptedVector(dt, t); - - builder->appendBinData(field, buf.size(), BinDataType::Encrypt, buf.data()); -} - -constexpr auto kIndexKeyId = "12345678-1234-9876-1234-123456789012"_sd; -constexpr auto kUserKeyId = "ABCDEFAB-1234-9876-1234-123456789012"_sd; -static UUID indexKeyId = uassertStatusOK(UUID::parse(kIndexKeyId.toString())); -static UUID userKeyId = uassertStatusOK(UUID::parse(kUserKeyId.toString())); - -std::vector<char> testValue = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19}; -std::vector<char> testValue2 = {0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29}; - -const FLEIndexKey& getIndexKey() { - static std::string indexVec = hexblob::decode( - "7dbfebc619aa68a659f64b8e23ccd21644ac326cb74a26840c3d2420176c40ae088294d00ad6cae9684237b21b754cf503f085c25cd320bf035c3417416e1e6fe3d9219f79586582112740b2add88e1030d91926ae8afc13ee575cfb8bb965b7"_sd); - static FLEIndexKey indexKey(KeyMaterial(indexVec.begin(), indexVec.end())); - return indexKey; -} - -const FLEUserKey& getUserKey() { - static std::string userVec = hexblob::decode( - "a7ddbc4c8be00d51f68d9d8e485f351c8edc8d2206b24d8e0e1816d005fbe520e489125047d647b0d8684bfbdbf09c304085ed086aba6c2b2b1677ccc91ced8847a733bf5e5682c84b3ee7969e4a5fe0e0c21e5e3ee190595a55f83147d8de2a"_sd); - static FLEUserKey userKey(KeyMaterial(userVec.begin(), userVec.end())); - return userKey; -} - - -BSONObj generateFFP(StringData path, int value) { - auto indexKey = getIndexKey(); - FLEIndexKeyAndId indexKeyAndId(indexKey.data, indexKeyId); - auto userKey = getUserKey(); - FLEUserKeyAndId userKeyAndId(userKey.data, indexKeyId); - - BSONObj doc = BSON("value" << value); - auto element = doc.firstElement(); - auto fpp = FLEClientCrypto::serializeFindPayload(indexKeyAndId, userKeyAndId, element, 0); - - BSONObjBuilder builder; - toEncryptedBinData(path, EncryptedBinDataType::kFLE2FindEqualityPayload, fpp, &builder); - return builder.obj(); -} - -class FLEServerHighCardRewriteTest : public unittest::Test { -public: - FLEServerHighCardRewriteTest() {} - - void setUp() override {} - - void tearDown() override {} - -protected: - BasicMockFLEQueryRewriter _mock; -}; - - -TEST_F(FLEServerHighCardRewriteTest, HighCard_TopLevel_Equality) { - _mock.setForceHighCardinalityForTest(); - - auto match = generateFFP("ssn", 1); - auto expected = fromjson(R"({ - "$expr": { - "$_internalFleEq": { - "field": "$ssn", - "edc": { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - }, - "counter": { - "$numberLong": "0" - }, - "server": { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - } - } -})"); - - auto actual = _mock.rewriteMatchExpressionForTest(match); - ASSERT_BSONOBJ_EQ(actual, expected); -} - - -TEST_F(FLEServerHighCardRewriteTest, HighCard_TopLevel_In) { - _mock.setForceHighCardinalityForTest(); - - auto ffp1 = generateFFP("ssn", 1); - auto ffp2 = generateFFP("ssn", 2); - auto ffp3 = generateFFP("ssn", 3); - auto expected = fromjson(R"({ - "$or": [ - { - "$expr": { - "$_internalFleEq": { - "field": "$ssn", - "edc": { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - }, - "counter": { - "$numberLong": "0" - }, - "server": { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - } - } - }, - { - "$expr": { - "$_internalFleEq": { - "field": "$ssn", - "edc": { - "$binary": { - "base64": "CLpCo6rNuYMVT+6n1HCX15MNrVYDNqf6udO46ayo43Sw", - "subType": "6" - } - }, - "counter": { - "$numberLong": "0" - }, - "server": { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - } - } - }, - { - "$expr": { - "$_internalFleEq": { - "field": "$ssn", - "edc": { - "$binary": { - "base64": "CPi44oCQHnNDeRqHsNLzbdCeHt2DK/wCly0g2dxU5fqN", - "subType": "6" - } - }, - "counter": { - "$numberLong": "0" - }, - "server": { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - } - } - } - ] -})"); - - auto match = - BSON("ssn" << BSON("$in" << BSON_ARRAY(ffp1.firstElement() - << ffp2.firstElement() << ffp3.firstElement()))); - - auto actual = _mock.rewriteMatchExpressionForTest(match); - ASSERT_BSONOBJ_EQ(actual, expected); -} - - -TEST_F(FLEServerHighCardRewriteTest, HighCard_TopLevel_Expr) { - - _mock.setForceHighCardinalityForTest(); - - auto ffp = generateFFP("$ssn", 1); - int len; - auto v = ffp.firstElement().binDataClean(len); - auto match = BSON("$expr" << BSON("$eq" << BSON_ARRAY(ffp.firstElement().fieldName() - << BSONBinData(v, len, Encrypt)))); - - auto expected = fromjson(R"({ "$expr": { - "$_internalFleEq": { - "field": "$ssn", - "edc": { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - }, - "counter": { - "$numberLong": "0" - }, - "server": { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - } - } - })"); - - auto actual = _mock.rewriteMatchExpressionForTest(match); - ASSERT_BSONOBJ_EQ(actual, expected); -} - -TEST_F(FLEServerHighCardRewriteTest, HighCard_TopLevel_Expr_In) { - - _mock.setForceHighCardinalityForTest(); - - auto ffp = generateFFP("$ssn", 1); - int len; - auto v = ffp.firstElement().binDataClean(len); - - auto ffp2 = generateFFP("$ssn", 1); - int len2; - auto v2 = ffp2.firstElement().binDataClean(len2); - - auto match = BSON( - "$expr" << BSON("$in" << BSON_ARRAY(ffp.firstElement().fieldName() - << BSON_ARRAY(BSONBinData(v, len, Encrypt) - << BSONBinData(v2, len2, Encrypt))))); - - auto expected = fromjson(R"({ "$expr": { "$or" : [ { - "$_internalFleEq": { - "field": "$ssn", - "edc": { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - }, - "counter": { - "$numberLong": "0" - }, - "server": { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - }}, - { - "$_internalFleEq": { - "field": "$ssn", - "edc": { - "$binary": { - "base64": "CEWSmQID7SfwyAUI3ZkSFkATKryDQfnxXEOGad5d4Rsg", - "subType": "6" - } - }, - "counter": { - "$numberLong": "0" - }, - "server": { - "$binary": { - "base64": "COuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/Ps", - "subType": "6" - } - } - }} - ]}})"); - - auto actual = _mock.rewriteMatchExpressionForTest(match); - ASSERT_BSONOBJ_EQ(actual, expected); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index f894629037f..18851f0ddb9 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -863,14 +863,6 @@ server_parameters: gt: 0 lt: 16777216 - internalQueryFLEAlwaysUseHighCardinalityMode: - description: "Boolean flag to force FLE to always use low selectivity mode" - set_at: [ startup, runtime ] - cpp_varname: "internalQueryFLEAlwaysUseHighCardinalityMode" - cpp_vartype: AtomicWord<bool> - default: - expr: false - # Note for adding additional query knobs: # # When adding a new query knob, you should consider whether or not you need to add an 'on_update' diff --git a/src/mongo/db/query/sbe_stage_builder_expression.cpp b/src/mongo/db/query/sbe_stage_builder_expression.cpp index 9fd4aba048e..d422d5a0dd7 100644 --- a/src/mongo/db/query/sbe_stage_builder_expression.cpp +++ b/src/mongo/db/query/sbe_stage_builder_expression.cpp @@ -377,7 +377,6 @@ public: void visit(const ExpressionLn* expr) final {} void visit(const ExpressionLog* expr) final {} void visit(const ExpressionLog10* expr) final {} - void visit(const ExpressionInternalFLEEqual* expr) final {} void visit(const ExpressionMap* expr) final {} void visit(const ExpressionMeta* expr) final {} void visit(const ExpressionMod* expr) final {} @@ -610,7 +609,6 @@ public: void visit(const ExpressionLn* expr) final {} void visit(const ExpressionLog* expr) final {} void visit(const ExpressionLog10* expr) final {} - void visit(const ExpressionInternalFLEEqual* expr) final {} void visit(const ExpressionMap* expr) final {} void visit(const ExpressionMeta* expr) final {} void visit(const ExpressionMod* expr) final {} @@ -2318,9 +2316,6 @@ public: _context->pushExpr( sbe::makeE<sbe::ELocalBind>(frameId, std::move(binds), std::move(log10Expr))); } - void visit(const ExpressionInternalFLEEqual* expr) final { - unsupportedExpression("$_internalFleEq"); - } void visit(const ExpressionMap* expr) final { unsupportedExpression("$map"); } diff --git a/src/mongo/db/s/collection_sharding_state.cpp b/src/mongo/db/s/collection_sharding_state.cpp index 03cb0dd10fa..b8fd37f965e 100644 --- a/src/mongo/db/s/collection_sharding_state.cpp +++ b/src/mongo/db/s/collection_sharding_state.cpp @@ -157,6 +157,10 @@ std::vector<NamespaceString> CollectionShardingState::getCollectionNames(Operati return collectionsMap->getCollectionNames(); } +void CollectionShardingState::clearFilteringMetadata_DoNotUseIt(OperationContext* opCtx) { + clearFilteringMetadata(opCtx); +} + void CollectionShardingStateFactory::set(ServiceContext* service, std::unique_ptr<CollectionShardingStateFactory> factory) { auto& collectionsMap = CollectionShardingStateMap::get(service); diff --git a/src/mongo/db/s/collection_sharding_state.h b/src/mongo/db/s/collection_sharding_state.h index 575ccb85f2f..e15af4cfb31 100644 --- a/src/mongo/db/s/collection_sharding_state.h +++ b/src/mongo/db/s/collection_sharding_state.h @@ -159,6 +159,19 @@ public: * Returns the number of ranges scheduled for deletion on the collection. */ virtual size_t numberOfRangesScheduledForDeletion() const = 0; + + /** + * Public interface to clear the filtering metadata associated to a namespace. Do not use it + * without the consent of the Sharding Team, please. + */ + void clearFilteringMetadata_DoNotUseIt(OperationContext* opCtx); + +private: + /** + * Private interface to clear the filtering metadata. Please, do not make it public. See + * clearFilteringMetadata_DoNotUseIt for more information. + */ + virtual void clearFilteringMetadata(OperationContext* opCtx) = 0; }; /** diff --git a/src/mongo/db/s/collection_sharding_state_factory_standalone.cpp b/src/mongo/db/s/collection_sharding_state_factory_standalone.cpp index 6422dc066f1..995a5f28c72 100644 --- a/src/mongo/db/s/collection_sharding_state_factory_standalone.cpp +++ b/src/mongo/db/s/collection_sharding_state_factory_standalone.cpp @@ -67,6 +67,9 @@ public: size_t numberOfRangesScheduledForDeletion() const override { return 0; } + +private: + void clearFilteringMetadata(OperationContext* opCtx) override {} }; } // namespace diff --git a/src/mongo/db/session_catalog_mongod.cpp b/src/mongo/db/session_catalog_mongod.cpp index 50b63c0f390..fbfa223d80c 100644 --- a/src/mongo/db/session_catalog_mongod.cpp +++ b/src/mongo/db/session_catalog_mongod.cpp @@ -379,55 +379,23 @@ void createTransactionTable(OperationContext* opCtx) { auto createCollectionStatus = storageInterface->createCollection( opCtx, NamespaceString::kSessionTransactionsTableNamespace, options); - auto internalTransactionsFlagEnabled = - feature_flags::gFeatureFlagInternalTransactions.isEnabled( - serverGlobalParams.featureCompatibility); - - // This flag is off by default and only exists to facilitate creating the partial index more - // easily, so we don't tie it to FCV. This overrides the internal transactions feature flag. - auto alwaysCreateIndexFlagEnabled = - feature_flags::gFeatureFlagAlwaysCreateConfigTransactionsPartialIndexOnStepUp - .isEnabledAndIgnoreFCV(); - if (createCollectionStatus == ErrorCodes::NamespaceExists) { - if (!internalTransactionsFlagEnabled && !alwaysCreateIndexFlagEnabled) { + if (!feature_flags::gFeatureFlagInternalTransactions.isEnabled( + serverGlobalParams.featureCompatibility)) { return; } - bool collectionIsEmpty = false; - { - AutoGetCollection autoColl( - opCtx, NamespaceString::kSessionTransactionsTableNamespace, LockMode::MODE_IS); - invariant(autoColl); - - if (autoColl->getIndexCatalog()->findIndexByName( - opCtx, MongoDSessionCatalog::kConfigTxnsPartialIndexName)) { - // Index already exists, so there's nothing to do. - return; - } - - collectionIsEmpty = autoColl->isEmpty(opCtx); - } - - if (!collectionIsEmpty) { - // Unless explicitly enabled, don't create the index to avoid delaying step up. - if (alwaysCreateIndexFlagEnabled) { - AutoGetCollection autoColl( - opCtx, NamespaceString::kSessionTransactionsTableNamespace, LockMode::MODE_X); - IndexBuildsCoordinator::get(opCtx)->createIndex( - opCtx, - autoColl->uuid(), - MongoDSessionCatalog::getConfigTxnPartialIndexSpec(), - IndexBuildsManager::IndexConstraints::kEnforce, - false /* fromMigration */); - } + AutoGetCollection autoColl( + opCtx, NamespaceString::kSessionTransactionsTableNamespace, LockMode::MODE_IS); + // During failover recovery it is possible that the collection is created, but the partial + // index is not since they are recorded as separate oplog entries. If it is already created + // or if the collection isn't empty we can return early. + if (autoColl->getIndexCatalog()->findIndexByName( + opCtx, MongoDSessionCatalog::kConfigTxnsPartialIndexName) || + !autoColl->isEmpty(opCtx)) { return; } - - // The index does not exist and the collection is empty, so fall through to create it on the - // empty collection. This can happen after a failover because the collection and index - // creation are recorded as separate oplog entries. } else { uassertStatusOKWithContext(createCollectionStatus, str::stream() @@ -436,7 +404,8 @@ void createTransactionTable(OperationContext* opCtx) { << " collection"); } - if (!internalTransactionsFlagEnabled && !alwaysCreateIndexFlagEnabled) { + if (!feature_flags::gFeatureFlagInternalTransactions.isEnabled( + serverGlobalParams.featureCompatibility)) { return; } diff --git a/src/mongo/idl/basic_types.idl b/src/mongo/idl/basic_types.idl index 634b05d9539..d124f29cdae 100644 --- a/src/mongo/idl/basic_types.idl +++ b/src/mongo/idl/basic_types.idl @@ -156,13 +156,6 @@ types: cpp_type: "std::array<std::uint8_t, 16>" deserializer: "mongo::BSONElement::uuid" - bindata_encrypt: - bson_serialization_type: bindata - bindata_subtype: encrypt - description: "A BSON bindata of encrypt sub type" - cpp_type: "std::vector<std::uint8_t>" - deserializer: "mongo::BSONElement::_binDataVector" - uuid: bson_serialization_type: bindata bindata_subtype: uuid diff --git a/src/mongo/idl/generic_argument.idl b/src/mongo/idl/generic_argument.idl index 5cb308bd2e1..fdf340cbef1 100644 --- a/src/mongo/idl/generic_argument.idl +++ b/src/mongo/idl/generic_argument.idl @@ -104,7 +104,6 @@ generic_argument_lists: mayBypassWriteBlocking: forward_to_shards: true - generic_reply_field_lists: generic_reply_fields_api_v1: description: "Fields that may appear in any command reply. These are guaranteed backwards-compatible for as long as the server supports API Version 1." diff --git a/src/mongo/s/write_ops/batch_write_op.cpp b/src/mongo/s/write_ops/batch_write_op.cpp index a61ee3dd4bf..b034bcd0ee4 100644 --- a/src/mongo/s/write_ops/batch_write_op.cpp +++ b/src/mongo/s/write_ops/batch_write_op.cpp @@ -289,13 +289,6 @@ void populateCollectionUUIDMismatch(OperationContext* opCtx, } } -int getEncryptionInformationSize(const BatchedCommandRequest& req) { - if (!req.getWriteCommandRequestBase().getEncryptionInformation()) { - return 0; - } - return req.getWriteCommandRequestBase().getEncryptionInformation().get().toBSON().objsize(); -} - } // namespace BatchWriteOp::BatchWriteOp(OperationContext* opCtx, const BatchedCommandRequest& clientRequest) @@ -428,7 +421,6 @@ Status BatchWriteOp::targetBatch( // // The constant 4 is chosen as the size of the BSON representation of the stmtId. const int writeSizeBytes = getWriteSizeBytes(writeOp) + - getEncryptionInformationSize(_clientRequest) + write_ops::kWriteCommandBSONArrayPerElementOverheadBytes + (_batchTxnNum ? write_ops::kWriteCommandBSONArrayPerElementOverheadBytes + 4 : 0); @@ -591,9 +583,6 @@ BatchedCommandRequest BatchWriteOp::buildBatchRequest(const TargetedWriteBatch& wcb.setOrdered(_clientRequest.getWriteCommandRequestBase().getOrdered()); wcb.setCollectionUUID(_clientRequest.getWriteCommandRequestBase().getCollectionUUID()); - wcb.setEncryptionInformation( - _clientRequest.getWriteCommandRequestBase().getEncryptionInformation()); - if (targeter.isShardedTimeSeriesBucketsNamespace() && !_clientRequest.getNS().isTimeseriesBucketsCollection()) { wcb.setIsTimeseriesNamespace(true); diff --git a/src/mongo/util/dns_query_posix-impl.h b/src/mongo/util/dns_query_posix-impl.h index 93431114f7f..8c39084deaf 100644 --- a/src/mongo/util/dns_query_posix-impl.h +++ b/src/mongo/util/dns_query_posix-impl.h @@ -188,7 +188,7 @@ public: uasserted(ErrorCodes::DNSProtocolError, "DNS CNAME record could not be decompressed"); } - return std::string(&buf[0]); + return std::string(&buf[0], length); } DNSQueryType getType() const { |