SERVER-65280 Create kill jscore passthrough suite that talks directly to shardsvrs

5 files changed, 220 insertions, 33 deletions

diff --git a/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_jscore_passthrough.yml
index 4aa38ff94dd..e31f260e4f8 100644
--- a/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_jscore_passthrough.yml
@@ -4,30 +4,6 @@ selector:
   roots:
   - jstests/core/**/*.js
   exclude_files:
-  # These tests run applyOps which is not supported in serverless.
-  - jstests/core/apply_ops1.js
-  - jstests/core/apply_ops1.js
-  - jstests/core/apply_ops2.js
-  - jstests/core/apply_ops_dups.js
-  - jstests/core/apply_ops_index_collation.js
-  - jstests/core/apply_ops_invalid_index_spec.js
-  - jstests/core/apply_ops_missing_field.js
-  - jstests/core/apply_ops_system_dot_views.js
-  - jstests/core/apply_ops_without_ns.js
-  - jstests/core/bypass_doc_validation.js
-  - jstests/core/collmod_without_uuid.js
-  - jstests/core/txns/commands_banning_txnnumber_outside_transactions.js
-  - jstests/core/txns/commands_not_allowed_in_txn.js
-  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js
-  - jstests/core/txns/statement_ids_accepted.js
-  - jstests/core/list_collections1.js
-  - jstests/core/list_collections_filter.js
-  - jstests/core/list_collections_no_views.js
-  - jstests/core/views/view_with_invalid_dbname.js
-  - jstests/core/views/views_creation.js
-  - jstests/core/views/invalid_system_views.js
-  - jstests/core/views/views_all_commands.js
-  - jstests/core/rename_collection_staytemp.js
   # This test expects that the connection (i.e. 'threadName') does not change throughout each test
   # case. That is not always true when the test driver use a replica set connection string. As this
   # test sets fail point with error code NotWritablePrimary, the test driver changes its topology to
diff --git a/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml
new file mode 100644
index 00000000000..7a1f8d9f47a
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml
@@ -0,0 +1,194 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/core/**/*.js
+  exclude_files:
+  # This test expects that the connection (i.e. 'threadName') does not change throughout each test
+  # case. That is not always true when the test driver use a replica set connection string. As this
+  # test sets fail point with error code NotWritablePrimary, the test driver changes its topology to
+  # ReplicaSetNoPrimary and create a new connection once it get the primary again.
+  - jstests/core/failcommand_failpoint.js
+
+  # Transactions do not support retryability of individual operations.
+  # TODO: Remove this once it is supported (SERVER-33952).
+  - jstests/core/txns/**/*.js
+
+  # No-op retries are not ignored by top.
+  - jstests/core/operation_latency_histogram.js
+
+  # Expect drops/creates to fail or have a certain response:
+  - jstests/core/explain_upsert.js
+  - jstests/core/indexes_multiple_commands.js
+
+  # Expect certain responses, but retries of successfully completed commands may return
+  # different values:
+  - jstests/core/create_indexes.js
+  - jstests/core/objid5.js
+
+  # Expect results to return in a certain order, secondaries may apply ops out of order.
+  - jstests/core/coveredIndex1.js
+  - jstests/core/sortc.js
+
+  # TODO SERVER-31242: findAndModify no-op retry should respect the fields option.
+  - jstests/core/find_and_modify.js
+  - jstests/core/find_and_modify2.js
+  - jstests/core/find_and_modify_server6865.js
+
+  - jstests/core/bench_test*.js # benchRun() used for writes
+  - jstests/core/benchrun_pipeline_updates.js # benchRun() used for writes
+  - jstests/core/explain_large_bounds.js # Stepdown can timeout waiting for global lock.
+  - jstests/core/insert2.js # Creates new mongo connection.
+
+  # Creates new mongo connection but won't retry connecting.
+  - jstests/core/shell_connection_strings.js
+
+  # The next two tests have no tag "requires_non_retryable_commands" but errors thrown by
+  # the network_error_and_txn_override.js override when it refuses to run a certain command.
+  # "Refusing to run a test that issues commands that are not blindly retryable"
+  - jstests/core/profile3.js
+  - jstests/core/list_collections_filter.js
+  # The next two tests have no tag "requires_non_retryable_writes" but errors thrown by
+  # the network_error_and_txn_override.js override when it refuses to run a certain command.
+  # "Refusing to run a test that issues non-retryable write operations since the test likely makes
+  # assertions on the write results and can lead to spurious failures if a network error occurs"
+  - jstests/core/crud_api.js
+  - jstests/core/views/views_stats.js
+  # The next one test has no tag "does_not_support_stepdowns" but errors thrown by
+  # the network_error_and_txn_override.js override when it refuses to run a certain command.
+  # "Refusing to run a test that issues commands that may return different values after a failover"
+  - jstests/core/fsync.js
+
+  # Expect failure to connect server with an empty connection string.
+  # But, network_error_and_txn_override.js override keeps retrying forever.
+  - jstests/core/connection_string_validation.js
+
+  exclude_with_any_tags:
+  - assumes_standalone_mongod
+  - command_not_supported_in_serverless
+  # capped collections are banned in Serverless.
+  - requires_capped
+  # In this suite the test talks directly to shardsvrs (doesn't go through mongos).
+  - directly_against_shardsvrs_incompatible
+  ##
+  # The next five tags correspond to the special errors thrown by the network_error_and_txn_override.js
+  # override when it refuses to run a certain command. Above each tag are the message(s) that cause
+  # the tag to be warranted.
+  ##
+  # "Refusing to run a test that issues a getMore command since if a network error occurs during
+  #   it then we won't know whether the cursor was advanced or not"
+  - requires_getmore
+  # "Refusing to run a test that issues non-retryable write operations since the test likely makes
+  #   assertions on the write results and can lead to spurious failures if a network error occurs"
+  - requires_non_retryable_writes
+  # "Refusing to run a test that issues commands that are not blindly retryable"
+  # "Refusing to run a test that issues an aggregation command with $out because it is not
+  #   retryable"
+  - requires_non_retryable_commands
+  # "Refusing to run a test that issues commands that may return different values after a failover"
+  # "Refusing to run a test that issues an aggregation command with explain because it may return
+  #   incomplete results"
+  # "Refusing to run a test that issues an aggregation command with $listLocalCursors or
+  #   $listLocalSessions because they rely on in-memory state that may not survive failovers"
+  # "Refusing to run a test that issues a mapReduce command, because it calls std::terminate() if
+  #   interrupted by a stepdown"
+  - does_not_support_stepdowns
+  # "Refusing to run a test that issues commands that may return different values after a failover"
+  - requires_profiling
+  # Operations in the main test shell aren't guaranteed to be causally consistent with operations
+  # performed earlier in a parallel shell if multiple nodes are electable because the latest
+  # operation and cluster times aren't shared between shells.
+  # "Cowardly refusing to run test with network retries enabled when it uses startParallelShell()"
+  # "Cowardly fail if startParallelShell is run with a mongod that had an unclean shutdown: ..."
+  - uses_parallel_shell
+  ##
+  # The next three tags correspond to the special errors thrown by the
+  # fail_unclean_shutdown_incompatible_commands.js override when it refuses to run commands that are
+  # inaccurate after an unclean shutdown. Above each tag is the message that causes the tag to be
+  # warranted.
+  ##
+  # "Cowardly fail if collStats is run with a mongod that had an unclean shutdown: ..."
+  - requires_collstats
+  # "Cowardly fail if dbStats is run with a mongod that had an unclean shutdown: ..."
+  - requires_dbstats
+  # "Cowardly fail if fastcount is run with a mongod that had an unclean shutdown: ..."
+  - requires_fastcount
+  ##
+  # The next two tags correspond to the special errors thrown by the
+  # set_read_and_write_concerns.js override when it refuses to replace the readConcern or
+  # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to be
+  # warranted.
+  ##
+  # "Cowardly refusing to override read concern of command: ..."
+  - assumes_read_concern_unchanged
+  # "writeConcern is not allowed within a multi-statement transaction"
+  - assumes_write_concern_unchanged
+  ## The next tag corresponds to long running-operations, as they may exhaust their number
+  # of retries and result in a network error being thrown.
+  - operations_longer_than_stepdown_interval
+  # Retrying a query can change whether a plan cache entry is active.
+  - inspects_whether_plan_cache_entry_is_active
+
+executor:
+  archive:
+    tests: true
+    hooks:
+      - CheckReplDBHash
+      - CheckReplOplogs
+      - ValidateCollections
+  config:
+    shell_options:
+      eval: >-
+        testingReplication = true;
+        load('jstests/libs/override_methods/network_error_and_txn_override.js');
+        db = connect(TestData.connectionString);
+        load("jstests/libs/override_methods/enable_sessions.js");
+        load("jstests/libs/override_methods/set_read_and_write_concerns.js");
+        load("jstests/libs/override_methods/fail_unclean_shutdown_incompatible_commands.js");
+        load("jstests/libs/override_methods/fail_unclean_shutdown_start_parallel_shell.js");
+      global_vars:
+        TestData:
+          alwaysInjectTransactionNumber: true
+          defaultReadConcernLevel: "majority"
+          logRetryAttempts: true
+          networkErrorAndTxnOverrideConfig:
+            retryOnNetworkErrors: true
+          overrideRetryAttempts: 3
+          sessionOptions:
+            readConcern:
+              level: "majority"
+            # Force DBClientRS to find the primary for non-write commands.
+            readPreference:
+              mode: "primary"
+            retryWrites: true
+      # We specify nodb so the shell used by each test will attempt to connect after loading the
+      # retry logic in auto_retry_on_network_error.js.
+      nodb: ""
+  hooks:
+  - class: ContinuousStepdown
+    kill: true
+  # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
+  # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
+  # validating the entire contents of the collection.
+  - class: CheckReplOplogs
+  - class: CheckReplDBHash
+  - class: ValidateCollections
+    shell_options:
+      global_vars:
+        TestData:
+          skipEnforceFastCountOnValidate: true
+  - class: CleanEveryN
+    n: 20
+  fixture:
+    class: TalkDirectlyToShardsvrsFixture
+    common_mongod_options:
+      set_parameters:
+        enableTestCommands: 1
+        enableElectionHandoff: 0
+    num_replica_sets: 1
+    num_nodes_per_replica_set: 3
+    use_replica_set_connection_string: true
+    all_nodes_electable: true
+    replset_config_options:
+      settings:
+        catchUpTimeoutMillis: 0
diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py
index 3199ea11109..abc0ec908ef 100644
--- a/buildscripts/resmokelib/testing/hooks/stepdown.py
+++ b/buildscripts/resmokelib/testing/hooks/stepdown.py
@@ -10,7 +10,7 @@ import pymongo.errors
 
 import buildscripts.resmokelib.utils.filesystem as fs
 from buildscripts.resmokelib import errors
-from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface
+from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface, talk_directly_to_shardsvrs
 from buildscripts.resmokelib.testing.fixtures import replicaset
 from buildscripts.resmokelib.testing.fixtures import shardedcluster
 from buildscripts.resmokelib.testing.fixtures import tenant_migration
@@ -144,6 +144,13 @@ class ContinuousStepdown(interface.Hook):  # pylint: disable=too-many-instance-a
             # Recursively call _add_fixture on all the independent clusters.
             for cluster_fixture in fixture.get_independent_clusters():
                 self._add_fixture(cluster_fixture)
+        elif isinstance(fixture, talk_directly_to_shardsvrs.TalkDirectlyToShardsvrsFixture):
+            if not fixture.all_nodes_electable:
+                raise ValueError(
+                    "The replica sets that are the target of the ContinuousStepdown hook must have"
+                    " the 'all_nodes_electable' option set.")
+            for rs_fixture in fixture.get_replsets():
+                self._rs_fixtures.append(rs_fixture)
 
 
 class FlagBasedStepdownLifecycle(object):
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml
index 79927a7c975..98b9c802a79 100644
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@@ -5058,14 +5058,6 @@ tasks:
       use_large_distro: "true"
 
 - <<: *gen_task_template
-  name: talk_directly_to_shardsvrs_jscore_passthrough_gen
-  tags: ["serverless"]
-  commands:
-  - func: "generate resmoke tasks"
-    vars:
-      use_large_distro: "true"
-
-- <<: *gen_task_template
   name: tenant_migration_causally_consistent_jscore_passthrough_gen
   tags: ["serverless"]
   commands:
@@ -5108,6 +5100,22 @@ tasks:
       fallback_num_sub_suites: 10
 
 - <<: *gen_task_template
+  name: talk_directly_to_shardsvrs_jscore_passthrough_gen
+  tags: ["serverless"]
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      use_large_distro: "true"
+
+- <<: *gen_task_template
+  name: talk_directly_to_shardsvrs_kill_primary_jscore_passthrough_gen
+  tags: ["serverless"]
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      use_large_distro: "true"
+
+- <<: *gen_task_template
   name: clustered_collection_passthrough_gen
   tags: ["large", "clustered_collections"]
   commands:
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index fa7f0fcc30e..8b5882ecd30 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -10,6 +10,8 @@
 //   uses_map_reduce_with_temp_collections,
 //   # Tenant migrations don't support applyOps.
 //   tenant_migration_incompatible,
+//   # Explain of a resolved view must be executed by mongos.
+//   directly_against_shardsvrs_incompatible,
 // ]
 
 /*