SERVER-67860 Support causally consistent passthrough for shard merge

1 files changed, 351 insertions, 0 deletions

diff --git a/buildscripts/resmokeconfig/suites/shard_merge_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/shard_merge_causally_consistent_jscore_passthrough.yml
new file mode 100644
index 00000000000..7a76f1b0bc3
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/shard_merge_causally_consistent_jscore_passthrough.yml
@@ -0,0 +1,351 @@
+config_variables:
+- &keyFile jstests/libs/authTestsKey
+- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
+- &authOptions
+  authenticationDatabase: admin
+  authenticationMechanism: SCRAM-SHA-256
+  password: *keyFileData
+  username: __system
+
+test_kind: multi_stmt_txn_passthrough
+# This suite starts two 3-node replica sets and uses the ContinuousShardMerge hook to run
+# background tenant migrations. It also wraps all CRUD commands in transactions and asserts that
+# the transactions are committed/aborted correctly while tenant migrations are running.
+
+selector:
+  roots:
+  - jstests/core/**/*.js
+
+  exclude_files:
+  #
+  # Excluded from tenant_migration_jscore_passthrough.yml
+  #
+
+  # These tests already run with transactions.
+  - jstests/core/txns/**/*.js
+  # Skip any tests that run with auth explicitly.
+  - jstests/core/*[aA]uth*.js
+
+  #
+  # Excluded from replica_sets_multi_stmt_txn_jscore_passthrough.yml
+  #
+
+  ##
+  ## Limitations with the way the runner file injects transactions.
+  ##
+
+  # These tests expects some statements to error, which will cause txns to abort entirely.
+  - jstests/core/api_params_getmore.js
+  - jstests/core/commands_with_uuid.js
+  - jstests/core/dbcase.js
+  - jstests/core/dbcase2.js
+  - jstests/core/explain_execution_error.js
+  - jstests/core/expr.js
+  - jstests/core/find9.js
+  - jstests/core/find_and_modify_invalid_query_params.js
+  - jstests/core/find_getmore_bsonsize.js
+  - jstests/core/find_getmore_cmd.js
+  - jstests/core/geo_allowedcomparisons.js
+  - jstests/core/geo_big_polygon2.js
+  - jstests/core/geonear_key.js
+  - jstests/core/in.js
+  - jstests/core/index8.js # No explicit check for failed command.
+  - jstests/core/index_decimal.js
+  - jstests/core/index_multiple_compatibility.js
+  - jstests/core/index_partial_write_ops.js
+  - jstests/core/indexa.js # No explicit check for failed command.
+  - jstests/core/indexes_multiple_commands.js
+  - jstests/core/js2.js
+  - jstests/core/json_schema/json_schema.js
+  - jstests/core/max_doc_size.js
+  - jstests/core/mr_bigobject.js
+  - jstests/core/not2.js
+  - jstests/core/null_query_semantics.js
+  - jstests/core/or1.js
+  - jstests/core/or2.js
+  - jstests/core/or3.js
+  - jstests/core/ord.js
+  - jstests/core/orj.js
+  - jstests/core/projection_expr_mod.js
+  - jstests/core/ref.js
+  - jstests/core/ref4.js
+  - jstests/core/regex_limit.js
+  - jstests/core/remove_undefined.js
+  - jstests/core/set7.js
+  - jstests/core/sortb.js
+  - jstests/core/sortf.js
+  - jstests/core/sortg.js
+  - jstests/core/sortj.js
+  - jstests/core/sort_with_meta_operator.js
+  - jstests/core/tailable_skip_limit.js
+  - jstests/core/type_array.js
+  - jstests/core/uniqueness.js
+  - jstests/core/unset2.js
+  - jstests/core/update_addToSet.js
+  - jstests/core/update_array_offset_positional.js
+  - jstests/core/update_find_and_modify_id.js
+  - jstests/core/update_modifier_pop.js
+  - jstests/core/updateh.js
+
+  # Reads from system.views.
+  - jstests/core/views/views_drop.js
+
+  ##
+  ## Some aggregation stages don't support snapshot readconcern.
+  ##
+
+  # explain (requires read concern local)
+  - jstests/core/agg_hint.js
+  - jstests/core/and.js
+  - jstests/core/collation.js
+  - jstests/core/distinct_multikey_dotted_path.js
+  - jstests/core/distinct_with_hashed_index.js
+  - jstests/core/explain_shell_helpers.js
+  - jstests/core/index_partial_read_ops.js
+  - jstests/core/optimized_match_explain.js
+  - jstests/core/sort_array.js
+  - jstests/core/views/views_collation.js
+  - jstests/core/wildcard_index_count.js
+  - jstests/core/explain_server_params.js
+
+  # $listSessions
+  - jstests/core/list_all_local_sessions.js
+  - jstests/core/list_all_sessions.js
+  - jstests/core/list_sessions.js
+
+  # $indexStats
+  - jstests/core/index_stats.js
+
+  # $collStats
+  - jstests/core/views/views_coll_stats.js
+
+  # Errors expected to happen in tests, which can cause transactions to get aborted.
+  # So when the test tries to inspect the documents it can be out of sync (relative
+  # to test run without multi statement transactions).
+  - jstests/core/doc_validation.js
+  - jstests/core/doc_validation_options.js
+  - jstests/core/field_name_validation.js
+  - jstests/core/insert_illegal_doc.js
+  - jstests/core/positional_projection.js
+  - jstests/core/push_sort.js
+  - jstests/core/update_dbref.js
+
+  # Trick for bypassing mongo shell validation in the test doesn't work because txn_override
+  # retry logic will hit the shell validation.
+  - jstests/core/invalid_db_name.js
+
+  # Multiple writes in a txn, some of which fail because the collection doesn't exist.
+  # We create the collection and retry the last write, but previous writes would have
+  # still failed.
+  - jstests/core/dbref1.js
+  - jstests/core/dbref2.js
+  - jstests/core/ref3.js
+  - jstests/core/update3.js
+
+  ##
+  ## Error: Unable to acquire lock within a max lock request timeout of '0ms' milliseconds
+  ##
+
+  # Collection drops done through applyOps are not converted to w:majority
+  - jstests/core/views/invalid_system_views.js
+
+  ##
+  ## Misc. reasons.
+  ##
+
+  # SERVER-34772 Tailable Cursors are not allowed with snapshot readconcern.
+  - jstests/core/awaitdata_getmore_cmd.js
+  - jstests/core/getmore_cmd_maxtimems.js
+  - jstests/core/tailable_cursor_invalidation.js
+  - jstests/core/tailable_getmore_batch_size.js
+
+  # Expects collection to not have been created
+  - jstests/core/insert_id_undefined.js
+
+  # Creates sessions explicitly, resulting in txns being run through different sessions
+  # using a single txnNumber.
+  - jstests/core/views/views_all_commands.js
+
+  # Fails with implicit sessions because it will use multiple sessions on the same Mongo connection.
+  - jstests/core/dropdb.js
+
+  # Committing a transaction when the server is fsync locked fails.
+  - jstests/core/fsync.js
+
+  # Expects legacy errors ($err).
+  - jstests/core/constructors.js
+
+  # txn interrupted by command outside of txn before getMore runs.
+  - jstests/core/commands_namespace_parsing.js
+  - jstests/core/drop3.js
+  - jstests/core/geo_s2cursorlimitskip.js
+  - jstests/core/getmore_invalidated_cursors.js
+  - jstests/core/getmore_invalidated_documents.js
+  - jstests/core/kill_cursors.js
+  - jstests/core/list_indexes.js
+  - jstests/core/list_indexes_invalidation.js
+  - jstests/core/list_namespaces_invalidation.js
+  - jstests/core/oro.js
+  - jstests/core/sort_with_update_between_getmores.js
+
+  # Parallel Shell - we do not signal the override to end a txn when a parallel shell closes.
+  - jstests/core/compact_keeps_indexes.js
+  - jstests/core/count10.js
+  - jstests/core/count_plan_summary.js
+  - jstests/core/coveredIndex3.js
+  - jstests/core/crud_ops_do_not_throw_locktimeout.js
+  - jstests/core/distinct3.js
+  - jstests/core/find_and_modify_concurrent_update.js
+  - jstests/core/geo_update_btree.js
+  - jstests/core/killop_drop_collection.js
+  - jstests/core/loadserverscripts.js
+  - jstests/core/mr_killop.js
+  - jstests/core/removeb.js
+  - jstests/core/removec.js
+  - jstests/core/shellstartparallel.js
+  - jstests/core/updatef.js
+
+  # Command expects to see result from parallel operation.
+  # E.g. Suppose the following sequence of events: op1, join() op2 in parallel shell, op3.
+  # op3 will still be using the snapshot from op1, and not see op2 at all.
+  - jstests/core/bench_test1.js
+  - jstests/core/benchrun_pipeline_updates.js
+  - jstests/core/cursora.js
+
+  # Does not support tojson of command objects.
+  - jstests/core/SERVER-23626.js
+
+  exclude_with_any_tags:
+  - assumes_standalone_mongod
+  # These tests run getMore commands which are not supported in the tenant migration passthrough.
+  - requires_getmore
+  # Due to background tenant migrations, operations in the main test shell are not guaranteed to
+  # be causally consistent with operations in a parallel shell. The reason is that
+  # TenantMigrationCommitted error is only thrown when the client does a write or a atClusterTime/
+  # afterClusterTime or linearlizable read. Therefore, one of shell may not be aware that the
+  # migration has occurred and would not forward the read/write command to the right replica set.
+  - uses_parallel_shell
+  # Profile settings are stored in-memory only so are not transferred to the recipient.
+  - requires_profiling
+  # capped collections are banned in Serverless
+  - requires_capped
+  # emptycapped command is blocked during tenant migration.
+  - requires_emptycapped
+  - tenant_migration_incompatible
+  # "Cowardly refusing to override read concern of command: ..."
+  - assumes_read_concern_unchanged
+  # "writeConcern is not allowed within a multi-statement transaction"
+  - assumes_write_concern_unchanged
+  # Transactions are not allowed to operate on capped collections.
+  - requires_capped
+  # Retrying a query can change whether a plan cache entry is active.
+  - inspects_whether_plan_cache_entry_is_active
+  # $out is not supported in transactions
+  - uses_$out
+  # Transaction-continuing commands must use the same API parameters as the first command, so tests
+  # that use API parameters cannot be run with transactions.
+  - uses_api_parameters
+  - does_not_support_transactions
+  # Full validation can cause ongoing queries to fail. This can affect the tenant migration
+  # process.
+  - uses_full_validation
+  # Multi-updates that conflict with tenant migration are not retried by inject_tenant_prefix.js.
+  - requires_multi_updates
+  - requires_timeseries # Tenant migration, transactions not supported
+  - requires_fastcount
+
+executor:
+  archive:
+    hooks:
+      - CheckReplOplogs
+      - CheckReplDBHash
+      - ValidateCollections
+  config:
+    shell_options:
+      eval: >-
+        testingReplication = true;
+        load('jstests/libs/override_methods/inject_tenant_prefix.js');
+        load('jstests/libs/override_methods/enable_sessions.js');
+        load('jstests/libs/override_methods/txn_passthrough_cmd_massage.js');
+        load('jstests/libs/override_methods/network_error_and_txn_override.js');
+        jsTest.authenticate(db.getMongo());
+      global_vars:
+        TestData: &TestData
+          # TODO SERVER-69034 : replace tenantId with tenantIds
+          tenantId: "tenantMigrationTenantId"
+          auth: true
+          authMechanism: SCRAM-SHA-256
+          keyFile: *keyFile
+          keyFileData: *keyFileData
+          roleGraphInvalidationIsFatal: true
+          networkErrorAndTxnOverrideConfig:
+            wrapCRUDinTransactions: true
+          sessionOptions:
+            # Tests in this suite only read from primaries and only one node is electable, so causal
+            # consistency is not required to read your own writes.
+            causalConsistency: false
+          # TODO SERVER-68643 No longer needed when we only use shard merge protocol.
+          useLocalDBForDBCheck: true
+          # Shard Merge does not support fastcount. When we call CheckTenantMigrationDBHash after
+          # each migration we would potentially fail when enforcing using the fast count during that
+          # process.
+          skipEnforceFastCountOnValidate: true
+  hooks:
+  - class: ContinuousShardMerge
+    shell_options:
+      global_vars:
+        TestData:
+          <<: *TestData
+          authOptions: *authOptions
+  # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
+  # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
+  # validating the entire contents of the collection.
+  - class: CheckReplOplogs
+    shell_options:
+      global_vars:
+        TestData: *TestData
+  - class: CheckReplDBHash
+    shell_options:
+      global_vars:
+        TestData: *TestData
+  - class: ValidateCollections
+    shell_options:
+      global_vars:
+        TestData: *TestData
+  - class: CleanEveryN
+    n: 1
+  fixture:
+    class: ShardMergeFixture
+    common_mongod_options:
+      set_parameters:
+        enableTestCommands: 1
+        failpoint.abortTenantMigrationBeforeLeavingBlockingState:
+            mode:
+              activationProbability: 0.5
+        failpoint.pauseTenantMigrationBeforeLeavingBlockingState:
+            mode: alwaysOn
+            data:
+              blockTimeMS: 250
+        # To avoid routing commands in each test incorrectly, the ContinuousShardMerge hook
+        # only runs donorForgetMigration against the donor of each migration when it is safe to do
+        # so. Therefore, the garbage collection delay doesn't need to be large.
+        tenantMigrationGarbageCollectionDelayMS: 1
+        ttlMonitorSleepSecs: 1
+        # Tenant migrations is not currently compatible with implicitly replicated retryable
+        # findAndModify images.
+        storeFindAndModifyImagesInSideCollection: false
+        minSnapshotHistoryWindowInSeconds: 30
+      tlsMode: allowTLS
+      tlsCAFile: jstests/libs/ca.pem
+      tlsAllowInvalidHostnames: ''
+      auth: ''
+      keyFile: *keyFile
+      syncdelay: 1
+    per_mongod_options:
+      # Each entry is for a different replica set's extra mongod options.
+      - tlsCertificateKeyFile: jstests/libs/rs0.pem
+      - tlsCertificateKeyFile: jstests/libs/rs1.pem
+    num_replica_sets: 2
+    num_nodes_per_replica_set: 3
+    use_replica_set_connection_string: true
+    auth_options: *authOptions