SERVER-31194 Add a version of retryable_writes_jscore_passthrough.yml with stepdowns

4 files changed, 159 insertions, 3 deletions

diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml
index 62ecd89e683..d7ca295ad1a 100644
--- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml
@@ -29,6 +29,9 @@ selector:
   # TODO SERVER-31245: Inserts to "system.indexes" bypass the check for retryability.
   - jstests/core/batch_write_command_insert.js
 
+  # TODO SERVER-31198: Remove once retry attempts are always decremented.
+  - jstests/core/write_result.js
+
 executor:
   config:
     shell_options:
diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml
new file mode 100644
index 00000000000..a52965222b6
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml
@@ -0,0 +1,142 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/core/**/*.js
+  exclude_files:
+  # These tests are not expected to pass with replica-sets:
+  - jstests/core/capped_update.js
+  - jstests/core/dbadmin.js
+  - jstests/core/opcounters_write_cmd.js
+  - jstests/core/read_after_optime.js
+
+  # No-op retries are not ignored by top, the profiler, or opcount.
+  - jstests/core/operation_latency_histogram.js
+  - jstests/core/profile2.js
+  - jstests/core/profile3.js
+  - jstests/core/profile_findandmodify.js
+  - jstests/core/top.js
+  - jstests/core/views/views_stats.js
+
+  # TODO SERVER-31249: getLastError should not be affected by no-op retries.
+  - jstests/core/bulk_legacy_enforce_gle.js
+
+  # TODO SERVER-31242: findAndModify no-op retry should respect the fields option.
+  - jstests/core/crud_api.js
+  - jstests/core/find_and_modify2.js
+  - jstests/core/find_and_modify_server6865.js
+
+  # TODO SERVER-31245: Inserts to "system.indexes" bypass the check for retryability.
+  - jstests/core/batch_write_command_insert.js
+
+  # Stepdown commands during fsync lock will fail.
+  - jstests/core/currentop.js
+  - jstests/core/fsync.js
+  - jstests/core/killop_drop_collection.js
+
+  # Expect drops/creates to fail or have certain a response:
+  - jstests/core/drop.js
+  - jstests/core/dropdb.js
+  - jstests/core/explain_upsert.js
+  - jstests/core/indexes_multiple_commands.js
+
+  # Expect certain responses, but retries of successfully completed commands may return
+  # different values:
+  - jstests/core/create_indexes.js
+  - jstests/core/objid5.js
+
+  # Expect results to return in a certain order, secondaries may apply ops out of order.
+  - jstests/core/coveredIndex1.js
+  - jstests/core/find1.js
+  - jstests/core/sortc.js
+
+  # Use $listLocalSessions aggregation stage command:
+  - jstests/core/list_all_local_cursors.js
+  - jstests/core/list_all_local_sessions.js
+  - jstests/core/list_local_sessions.js
+
+  - jstests/core/bench_test*.js # benchRun() used for writes
+  - jstests/core/connection_string_validation.js # Does not expect a replica set connection string.
+  - jstests/core/explain_large_bounds.js # Stepdown can timeout waiting for global lock.
+  - jstests/core/insert2.js # Creates new mongo connection.
+  - jstests/core/list_collections_filter.js # Temporary collections are dropped on failover.
+  - jstests/core/startup_log.js # Checks pid, which is different on each server.
+  - jstests/core/validate_cmd_ns.js # Calls _exec() directly, not retryable.
+
+  exclude_with_any_tags:
+  - cannot_inject_read_write_concern
+
+  # The next four tags correspond to the special errors thrown by the auto_retry_on_network_error.js
+  # override when it refuses to run a certain command. Above each tag are the messages that
+  # warrant it.
+
+  # "Refusing to run a test that issues a getMore command since if a network error occurs during
+  #   it then we won't know whether the cursor was advanced or not"
+  - requires_getmore
+
+  # "Refusing to run a test that issues non-retryable write operations since the test likely makes
+  #   assertions on the write results and can lead to spurious failures if a network error occurs"
+  - requires_non_retryable_writes
+
+  # "Refusing to run a test that issues commands that are not blindly retryable"
+  # "Refusing to run a test that issues an aggregation command with $out because it is not
+  #   retryable"
+  - requires_non_retryable_commands
+
+  # "Refusing to run a test that issues commands that may return different values after a failover"
+  # "Refusing to run a test that issues an aggregation command with explain because it may return
+  #   incomplete results"
+  # "Refusing to run a test that issues a mapReduce command, because it calls std::terminate() if
+  #   interrupted by a stepdown"
+  - does_not_support_stepdowns
+
+executor:
+  config:
+    shell_options:
+      eval: >-
+        testingReplication = true;
+        load("jstests/libs/override_methods/auto_retry_on_network_error.js");
+        db = connect(TestData.connectionString);
+        load("jstests/libs/override_methods/enable_sessions.js");
+        load("jstests/libs/override_methods/set_read_and_write_concerns.js");
+      global_vars:
+        TestData:
+          alwaysInjectTransactionNumber: true
+          defaultReadConcernLevel: "majority"
+          logRetryAttempts: true
+          overrideRetryAttempts: 3
+          sessionOptions:
+            readConcern:
+              level: "majority"
+            # Force DBClientRS to find the primary for non-write commands.
+            readPreference:
+              mode: "primary"
+            retryWrites: true
+      # We specify nodb so the shell used by each test will attempt to connect after loading the
+      # retry logic in auto_retry_on_network_error.js.
+      nodb: ""
+      readMode: commands
+  hooks:
+  - class: ContinuousStepdown
+  # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
+  # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
+  # validating the entire contents of the collection.
+  - class: CheckReplOplogs
+  - class: CheckReplDBHash
+  - class: ValidateCollections
+  - class: CleanEveryN
+    n: 20
+  fixture:
+    class: ReplicaSetFixture
+    mongod_options:
+      enableMajorityReadConcern: ''
+      oplogSize: 511
+      set_parameters:
+        enableTestCommands: 1
+        numInitialSyncAttempts: 1
+    all_nodes_electable: true
+    num_nodes: 3
+    replset_config_options:
+      settings:
+        # Speeds up failover, reduces chance DBClientRS times out retargeting the primary.
+        electionTimeoutMillis: 6500
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 097e0d7d1b4..ec7522dedc8 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -201,6 +201,12 @@ def mongo_shell_program(logger, executable=None, connection_string=None, filenam
     if "eval_prepend" in kwargs:
         eval_sb.append(str(kwargs.pop("eval_prepend")))
 
+    # If nodb is specified, pass the connection string through TestData so it can be used inside the
+    # test, then delete it so it isn't given as an argument to the mongo shell.
+    if "nodb" in kwargs and connection_string is not None:
+        test_data["connectionString"] = connection_string
+        connection_string = None
+
     for var_name in global_vars:
         _format_shell_vars(eval_sb, var_name, global_vars[var_name])
 
diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py
index 8ad70794807..438f4208cad 100644
--- a/buildscripts/resmokelib/testing/hooks/stepdown.py
+++ b/buildscripts/resmokelib/testing/hooks/stepdown.py
@@ -135,7 +135,9 @@ class _StepdownThread(threading.Thread):
             now = time.time()
             if now - self._last_exec > self._stepdown_interval_secs:
                 self._step_down_all()
-                self._last_exec = now
+                # Wait until each replica set has a primary, so the test can make progress.
+                self._await_primaries()
+                self._last_exec = time.time()
             now = time.time()
             # 'wait_secs' is used to wait 'self._stepdown_interval_secs' from the moment the last
             # stepdown command was sent.
@@ -158,8 +160,7 @@ class _StepdownThread(threading.Thread):
         # Wait until we are no longer executing stepdowns.
         self._is_idle_evt.wait()
         # Wait until we all the replica sets have primaries.
-        for fixture in self._rs_fixtures:
-            fixture.get_primary()
+        self._await_primaries()
 
     def resume(self):
         """Resumes the thread."""
@@ -177,6 +178,10 @@ class _StepdownThread(threading.Thread):
         # Wait until stop or timeout.
         self._is_stopped_evt.wait(timeout)
 
+    def _await_primaries(self):
+        for fixture in self._rs_fixtures:
+            fixture.get_primary()
+
     def _step_down_all(self):
         self._is_idle_evt.clear()
         for rs_fixture in self._rs_fixtures: