diff options
14 files changed, 0 insertions, 401 deletions
diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py index 2b40867519f..8d21fb3a31d 100644 --- a/buildscripts/idl/idl_check_compatibility.py +++ b/buildscripts/idl/idl_check_compatibility.py @@ -226,7 +226,6 @@ IGNORE_STABLE_TO_UNSTABLE_LIST: List[str] = [ 'aggregate-param-needsMerge', 'aggregate-param-fromMongos', # Bulk fixes for fields that are strictly internal all along and should thus be marked unstable. - 'aggregate-param-$_generateV2ResumeTokens', 'endSessions-param-txnNumber', 'endSessions-param-txnUUID', 'findAndModify-param-stmtId', @@ -320,7 +319,6 @@ ALLOWED_STABLE_FIELDS_LIST: List[str] = [ 'update-param-isTimeseriesNamespace', 'delete-param-isTimeseriesNamespace', 'findAndModify-param-stmtId', - 'aggregate-param-$_generateV2ResumeTokens', 'hello-param-loadBalanced', 'hello-reply-serviceId', 'hello-reply-isImplicitDefaultMajorityWC', diff --git a/buildscripts/resmokeconfig/suites/change_streams_v1_resume_token_passthrough.yml b/buildscripts/resmokeconfig/suites/change_streams_v1_resume_token_passthrough.yml deleted file mode 100644 index f94799973e2..00000000000 --- a/buildscripts/resmokeconfig/suites/change_streams_v1_resume_token_passthrough.yml +++ /dev/null @@ -1,70 +0,0 @@ -test_kind: js_test -selector: - roots: - - jstests/change_streams/**/*.js - exclude_files: - # This test explicitly compares v1 and v2 tokens, and must be able to generate the former. - - jstests/change_streams/generate_v1_resume_token.js - - # The following tests run in a sharded fixture where the mongod generates a new shard detected - # internal event, which needs to be swallowed by the mongos. This is not supported here, because - # this suite will return the event op name as 'kNewShardDetected', but the mongos expects the - # event op name to be 'migrateChunkToNewShard'. - - jstests/change_streams/create_event_from_chunk_migration.js - - jstests/change_streams/migrate_last_chunk_from_shard_event.js - - jstests/change_streams/oplog_rewrite/match_pushdown_namespace_rewrite_with_expanded_events.js - - jstests/change_streams/projection_fakes_internal_event.js - # The following test uses the '$changeStreamSplitLargeEvents' stage which requires v2 token. - - jstests/change_streams/split_large_event.js - - exclude_with_any_tags: - ## - # The next tags correspond to the special errors thrown by the - # set_read_and_write_concerns.js override when it refuses to replace the readConcern or - # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to be - # warranted. - ## - # "Cowardly refusing to override write concern of command: ..." - - assumes_write_concern_unchanged - -executor: - archive: - hooks: - - CheckReplDBHash - - CheckReplOplogs - - ValidateCollections - config: - shell_options: - global_vars: - TestData: - defaultReadConcernLevel: null - enableMajorityReadConcern: '' - # Enable causal consistency for change streams suites using 1 node replica sets. Some tests - # rely on the assumption that a w:majority write will be visible immediately in a subsequently - # opened change stream. In 1 node replica sets, an operation that majority commits at - # timestamp T will force the majority snapshot to advance to T, but the oplog visibility point - # may not have advanced to T yet. Subsequent majority snapshot reads will see this write in - # the oplog, but speculative majority reads may not, since they read from a local snapshot and - # are bound to the oplog visibility rules. Using causal consistency forces the visibility - # point to advance to the timestamp of the last write before doing a new read. - eval: >- - var testingReplication = true; - load('jstests/libs/override_methods/set_read_and_write_concerns.js'); - load('jstests/libs/override_methods/enable_causal_consistency_without_read_pref.js'); - load('jstests/libs/override_methods/implicit_v1_resume_token_changestreams.js'); - hooks: - # The CheckReplDBHash hook waits until all operations have replicated to and have been applied - # on the secondaries, so we run the ValidateCollections hook after it to ensure we're - # validating the entire contents of the collection. - - class: CheckReplOplogs - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ReplicaSetFixture - mongod_options: - bind_ip_all: '' - set_parameters: - enableTestCommands: 1 - num_nodes: 2 diff --git a/buildscripts/resmokeconfig/suites/change_streams_v1_resume_token_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/change_streams_v1_resume_token_sharded_collections_passthrough.yml deleted file mode 100644 index d91b200e1fb..00000000000 --- a/buildscripts/resmokeconfig/suites/change_streams_v1_resume_token_sharded_collections_passthrough.yml +++ /dev/null @@ -1,63 +0,0 @@ -test_kind: js_test -selector: - roots: - - jstests/change_streams/**/*.js - exclude_files: - # This test explicitly compares v1 and v2 tokens, and must be able to generate the former. - - jstests/change_streams/generate_v1_resume_token.js - # This test uses the '$changeStreamSplitLargeEvents' stage which requires v2 token. - - jstests/change_streams/split_large_event.js - - exclude_with_any_tags: - ## - # The next tags correspond to the special errors thrown by the - # set_read_and_write_concerns.js override when it refuses to replace the readConcern or - # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to be - # warranted. - ## - # "Cowardly refusing to override write concern of command: ..." - - assumes_write_concern_unchanged - # Exclude any that assume sharding is disabled - - assumes_against_mongod_not_mongos - - assumes_unsharded_collection - -executor: - archive: - hooks: - - CheckReplDBHash - - ValidateCollections - config: - shell_options: - global_vars: - TestData: - defaultReadConcernLevel: null - enableMajorityReadConcern: '' - # Enable causal consistency for change streams suites using 1 node replica sets. See - # change_streams.yml for detailed explanation. - eval: >- - var testingReplication = true; - load('jstests/libs/override_methods/set_read_and_write_concerns.js'); - load('jstests/libs/override_methods/implicitly_shard_accessed_collections.js'); - load('jstests/libs/override_methods/enable_causal_consistency_without_read_pref.js'); - load('jstests/libs/override_methods/implicit_v1_resume_token_changestreams.js'); - hooks: - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - mongos_options: - bind_ip_all: '' - set_parameters: - enableTestCommands: 1 - mongod_options: - bind_ip_all: '' - set_parameters: - enableTestCommands: 1 - writePeriodicNoops: 1 - periodicNoopIntervalSecs: 1 - coordinateCommitReturnImmediatelyAfterPersistingDecision: true - num_shards: 2 - enable_sharding: - - test diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 84d126de64b..6099ec98a80 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1308,7 +1308,6 @@ buildvariants: - name: change_streams_multi_stmt_txn_mongos_passthrough - name: change_streams_multi_stmt_txn_sharded_collections_passthrough - name: change_streams_per_shard_cursor_passthrough - - name: change_streams_v1_resume_token_sharded_collections_passthrough - name: fle2_sharding_high_cardinality - name: fle2_sharding - name: jstestfuzz_sharded_causal_consistency_gen diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 471c634d7b6..22b460ff425 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -4723,13 +4723,6 @@ tasks: - func: "do setup" - func: "run tests" -- <<: *task_template - name: change_streams_v1_resume_token_passthrough - tags: ["change_streams"] - commands: - - func: "do setup" - - func: "run tests" - - <<: *gen_task_template name: change_streams_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] @@ -4765,15 +4758,6 @@ tasks: - func: "do setup" - func: "run tests" -- <<: *task_template - name: change_streams_v1_resume_token_sharded_collections_passthrough - tags: ["change_streams"] - depends_on: - - name: change_streams - commands: - - func: "do setup" - - func: "run tests" - - <<: *gen_task_template name: change_streams_sharded_collections_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] diff --git a/jstests/aggregation/api_version_stage_allowance_checks.js b/jstests/aggregation/api_version_stage_allowance_checks.js index 3e6ec804b69..50b40ce4a3a 100644 --- a/jstests/aggregation/api_version_stage_allowance_checks.js +++ b/jstests/aggregation/api_version_stage_allowance_checks.js @@ -140,18 +140,6 @@ result = testDB.runCommand({ }); assert.commandWorked(result); -// Tests that the internal '$_generateV2ResumeTokens' option does not fail with 'apiStrict: true'. -result = testDB.runCommand({ - aggregate: collName, - pipeline: [{$project: {_id: 0}}], - cursor: {}, - writeConcern: {w: "majority"}, - $_generateV2ResumeTokens: false, - apiVersion: "1", - apiStrict: true -}); -assert.commandWorked(result); - // Tests that time-series collection can be queried (invoking $_internalUnpackBucket stage) // from an external client with 'apiStrict'. (function testInternalUnpackBucketAllowance() { diff --git a/jstests/change_streams/generate_v1_resume_token.js b/jstests/change_streams/generate_v1_resume_token.js deleted file mode 100644 index d3600ac1f80..00000000000 --- a/jstests/change_streams/generate_v1_resume_token.js +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Test that the $_generateV2ResumeTokens parameter can be used to force change streams to return v1 - * tokens. - * @tags: [ - * requires_fcv_61 - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/collection_drop_recreate.js"); // For assertDropAndRecreateCollection. - -const coll = assertDropAndRecreateCollection(db, jsTestName()); - -// Create one stream that returns v2 tokens, the default. -const v2Stream = coll.watch([]); - -// Create a second stream that explicitly requests v1 tokens. -const v1Stream = coll.watch([], {$_generateV2ResumeTokens: false}); - -// Insert a test document into the collection. -assert.commandWorked(coll.insert({_id: 1})); - -// Wait until both streams have encountered the insert operation. -assert.soon(() => v1Stream.hasNext() && v2Stream.hasNext()); -const v1Event = v1Stream.next(); -const v2Event = v2Stream.next(); - -// Confirm that the streams see the same event, but the resume tokens differ. -const v1ResumeToken = v1Event._id; -const v2ResumeToken = v2Event._id; - -delete v1Event._id; -delete v2Event._id; - -assert.docEq(v1Event, v2Event); -assert.neq(v1ResumeToken, v2ResumeToken, {v1ResumeToken, v2ResumeToken}); -})(); diff --git a/jstests/libs/override_methods/implicit_v1_resume_token_changestreams.js b/jstests/libs/override_methods/implicit_v1_resume_token_changestreams.js deleted file mode 100644 index f1e42abc0ba..00000000000 --- a/jstests/libs/override_methods/implicit_v1_resume_token_changestreams.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Loading this file overrides 'runCommand' with a function that modifies any $changeStream - * aggregation to use $_generateV2ResumeTokens:false. - */ -(function() { -"use strict"; - -load("jstests/libs/override_methods/override_helpers.js"); // For 'OverrideHelpers'. - -// Override runCommand to set $_generateV2ResumeTokens on all $changeStreams. -function runCommandV1Tokens(conn, dbName, cmdName, cmdObj, originalRunCommand, makeRunCommandArgs) { - if (OverrideHelpers.isAggregationWithChangeStreamStage(cmdName, cmdObj)) { - // Make a copy to avoid mutating the user's original command object. - cmdObj = Object.assign({}, cmdObj, {$_generateV2ResumeTokens: false}); - } - return originalRunCommand.apply(conn, makeRunCommandArgs(cmdObj)); -} - -// Always apply the override if a test spawns a parallel shell. -OverrideHelpers.prependOverrideInParallelShell( - "jstests/libs/override_methods/implicit_v1_resume_token_changestreams.js"); - -// Override the default runCommand with our custom version. -OverrideHelpers.overrideRunCommand(runCommandV1Tokens); -})(); diff --git a/jstests/noPassthrough/change_stream_generate_v2_tokens_flag_with_test_commands_disabled.js b/jstests/noPassthrough/change_stream_generate_v2_tokens_flag_with_test_commands_disabled.js deleted file mode 100644 index 8e71eb0a14e..00000000000 --- a/jstests/noPassthrough/change_stream_generate_v2_tokens_flag_with_test_commands_disabled.js +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Test that the $_generateV2ResumeTokens parameter cannot be used on mongoS when test commands are - * disabled. - * @tags: [ - * uses_change_streams, - * requires_sharding, - * requires_replication, - * ] - */ -(function() { -"use strict"; - -// Signal to the ShardingTest that we want to disable test commands. -TestData.enableTestCommands = false; - -// Create a sharding fixture with test commands disabled. -const st = new ShardingTest({shards: 1, rs: {nodes: 1}}); - -// Confirm that attempting to set any values for $_generateV2ResumeTokens field fails on mongos. -assert.throwsWithCode(() => st.s.watch([], {$_generateV2ResumeTokens: true}).hasNext(), 6528201); -assert.throwsWithCode(() => st.s.watch([], {$_generateV2ResumeTokens: false}).hasNext(), 6528201); - -// Confirm that attempting to run change streams with $_generateV2ResumeTokens:true fails on shards. -assert.throwsWithCode( - () => st.rs0.getPrimary().watch([], {$_generateV2ResumeTokens: true}).hasNext(), 6528200); - -// Explicity requesting v1 tokens is allowed on a shard. This is to allow a 6.0 mongoS to -// communicate with a 7.0 shard. -const stream = st.rs0.getPrimary().watch([], {$_generateV2ResumeTokens: false}); -assert.commandWorked(st.s.getDB("test")["coll"].insert({x: 1})); -assert.soon(() => stream.hasNext()); - -st.stop(); -})(); diff --git a/jstests/noPassthrough/change_stream_mongos_with_generate_v2_resume_tokens_flag.js b/jstests/noPassthrough/change_stream_mongos_with_generate_v2_resume_tokens_flag.js deleted file mode 100644 index 04dff09c8ed..00000000000 --- a/jstests/noPassthrough/change_stream_mongos_with_generate_v2_resume_tokens_flag.js +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Test that mongoS does not set the value of $_generateV2ResumeTokens on the commands it sends to - * the shards, if no value was specified by the client. If a value was specified, mongoS forwards it - * to the shards. On a replica set, no explicit value is set; the aggregation simply treats it as - * default-true. - * @tags: [ - * uses_change_streams, - * requires_sharding, - * requires_replication, - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/collection_drop_recreate.js"); // For assertDropAndRecreateCollection. -load("jstests/libs/profiler.js"); // For profilerHasSingleMatchingEntryOrThrow. - -// Create a sharding fixture with a single one-node replset shard and a one-node replset config -// server. The latter is to ensure that there is only one node that the internal new-shard monitor -// $changeStream can be sent to, since it is dispatched with secondaryPreferred readPreference. -const st = new ShardingTest({shards: 1, rs: {nodes: 1}, config: {nodes: 1}}); - -const mongosDB = st.s.getDB("test"); -const shardDB = st.rs0.getPrimary().getDB(mongosDB.getName()); -const configDB = st.configRS.getPrimary().getDB("config"); - -const mongosColl = assertDropAndRecreateCollection(mongosDB, jsTestName()); -const shardColl = shardDB[mongosColl.getName()]; -const configColl = configDB.shards; - -// Enable profiling on the shard and config server. -assert.commandWorked(shardDB.setProfilingLevel(2)); -assert.commandWorked(configDB.setProfilingLevel(2)); - -// Create one stream on mongoS that returns v2 tokens, the default. -const v2MongosStream = mongosColl.watch([], {comment: "v2MongosStream"}); - -// Create a second stream on mongoS that explicitly requests v1 tokens. -const v1MongosStream = - mongosColl.watch([], {comment: "v1MongosStream", $_generateV2ResumeTokens: false}); - -// Create a stream directly on the shard which returns the default v2 tokens. -const v2ShardStream = shardColl.watch([], {comment: "v2ShardStream"}); - -// Insert a test document into the collection. -assert.commandWorked(mongosColl.insert({_id: 1})); - -// Wait until all streams have encountered the insert operation. -assert.soon(() => v1MongosStream.hasNext() && v2MongosStream.hasNext() && v2ShardStream.hasNext()); - -// Confirm that in a sharded cluster, when v1 token is explicitly requested, mongoS fowards -// $_generateV2ResumeTokens:false to the shard. -profilerHasAtLeastOneMatchingEntryOrThrow({ - profileDB: shardDB, - filter: { - "originatingCommand.aggregate": mongosColl.getName(), - "originatingCommand.comment": "v1MongosStream", - "originatingCommand.$_generateV2ResumeTokens": false - } -}); - -// Confirm that we also set $_generateV2ResumeTokens to false on the internal new-shard monitoring -// $changeStream that we dispatch to the config servers. -profilerHasAtLeastOneMatchingEntryOrThrow({ - profileDB: configDB, - filter: { - "originatingCommand.aggregate": configColl.getName(), - "originatingCommand.comment": "v1MongosStream", - "originatingCommand.$_generateV2ResumeTokens": false - } -}); - -// Confirm that mongoS never sets the $_generateV2ResumeTokens field when client didn't explicitly -// specify. -profilerHasAtLeastOneMatchingEntryOrThrow({ - profileDB: shardDB, - filter: { - "originatingCommand.aggregate": mongosColl.getName(), - "originatingCommand.comment": "v2MongosStream", - "originatingCommand.$_generateV2ResumeTokens": {$exists: false} - } -}); - -// Confirm that we also do not set the $_generateV2ResumeTokens field on the request sent to the -// config server. -profilerHasAtLeastOneMatchingEntryOrThrow({ - profileDB: configDB, - filter: { - "originatingCommand.aggregate": configColl.getName(), - "originatingCommand.comment": "v2MongosStream", - "originatingCommand.$_generateV2ResumeTokens": {$exists: false} - } -}); - -// Confirm that on a replica set - in this case, a direct connection to the shard - no value is set -// for $_generateV2ResumeTokens if the client did not specify one. The aggregation defaults to -// treating the value as true. -profilerHasAtLeastOneMatchingEntryOrThrow({ - profileDB: shardDB, - filter: { - "originatingCommand.aggregate": mongosColl.getName(), - "originatingCommand.comment": "v2ShardStream", - "originatingCommand.$_generateV2ResumeTokens": {$exists: false} - } -}); - -st.stop(); -})(); diff --git a/src/mongo/db/commands/run_aggregate.cpp b/src/mongo/db/commands/run_aggregate.cpp index 9d67caf2df3..b03eae16dc9 100644 --- a/src/mongo/db/commands/run_aggregate.cpp +++ b/src/mongo/db/commands/run_aggregate.cpp @@ -451,15 +451,6 @@ boost::intrusive_ptr<ExpressionContext> makeExpressionContext( allowDiskUseByDefault.load()); expCtx->tempDir = storageGlobalParams.dbpath + "/_tmp"; expCtx->collationMatchesDefault = collationMatchesDefault; - - // If the request explicitly specified NOT to use v2 resume tokens for change streams, set this - // on the expCtx. This can happen if a the request originated from 6.0 mongos, or in test mode. - if (request.getGenerateV2ResumeTokens().has_value()) { - // We only ever expect an explicit $_generateV2ResumeTokens to be false. - uassert(6528200, "Invalid request for v2 tokens", !request.getGenerateV2ResumeTokens()); - expCtx->changeStreamTokenVersion = 1; - } - return expCtx; } diff --git a/src/mongo/db/pipeline/aggregate_command.idl b/src/mongo/db/pipeline/aggregate_command.idl index f825b983444..b53ea540f8e 100644 --- a/src/mongo/db/pipeline/aggregate_command.idl +++ b/src/mongo/db/pipeline/aggregate_command.idl @@ -283,12 +283,6 @@ commands: cpp_name: passthroughToShard optional: true stability: unstable - # TODO SERVER-65369: $_generateV2ResumeTokens can be removed after 7.0. - $_generateV2ResumeTokens: - description: "Internal parameter to signal whether v2 resume tokens should be generated." - type: optionalBool - cpp_name: generateV2ResumeTokens - stability: internal encryptionInformation: description: "Encryption Information schema and other tokens for CRUD commands" type: EncryptionInformation diff --git a/src/mongo/db/pipeline/sharded_agg_helpers.cpp b/src/mongo/db/pipeline/sharded_agg_helpers.cpp index bc045deb2f8..57146deb266 100644 --- a/src/mongo/db/pipeline/sharded_agg_helpers.cpp +++ b/src/mongo/db/pipeline/sharded_agg_helpers.cpp @@ -118,13 +118,6 @@ RemoteCursor openChangeStreamNewShardMonitor(const boost::intrusive_ptr<Expressi aggReq.setFromMongos(true); aggReq.setNeedsMerge(true); - // TODO SERVER-65369: This code block can be removed after 7.0. - if (isMongos() && expCtx->changeStreamTokenVersion == 1) { - // A request for v1 resume tokens on mongos should only be allowed in test mode. - tassert(6497000, "Invalid request for v1 resume tokens", getTestCommandsEnabled()); - aggReq.setGenerateV2ResumeTokens(false); - } - SimpleCursorOptions cursor; cursor.setBatchSize(0); aggReq.setCursor(cursor); diff --git a/src/mongo/s/query/cluster_aggregate.cpp b/src/mongo/s/query/cluster_aggregate.cpp index 02a41e64b17..6c9351efe57 100644 --- a/src/mongo/s/query/cluster_aggregate.cpp +++ b/src/mongo/s/query/cluster_aggregate.cpp @@ -139,16 +139,6 @@ boost::intrusive_ptr<ExpressionContext> makeExpressionContext( mergeCtx->inMongos = true; - // If the request explicity specified NOT to use v2 resume tokens for change streams, set this - // on the expCtx. We only ever expect to see an explicit value during testing. - if (request.getGenerateV2ResumeTokens().has_value()) { - // If $_generateV2ResumeTokens was specified, we must be testing and it must be false. - uassert(6528201, - "Invalid request for v2 resume tokens", - getTestCommandsEnabled() && !request.getGenerateV2ResumeTokens()); - mergeCtx->changeStreamTokenVersion = 1; - } - // Serialize the 'AggregateCommandRequest' and save it so that the original command can be // reconstructed for dispatch to a new shard, which is sometimes necessary for change streams // pipelines. |