SERVER-40335 Add waitForStepDownOnNonCommandShutdown parameter

When ReplSetTest.stopSet() kills the primary with SIGTERM, it must wait for the
primary to finish election handoff or, in cases where there is no candidate,
for the 10-second election handoff timeout to expire. Add a parameter and use
it in ReplSetTest.stopSet() to end tests quickly.

Disable waitForStepDownOnNonCommandShutdown for replica sets started directly
by resmoke.py so those sets shutdown quickly, too.

3 files changed, 31 insertions, 1 deletions

diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 3865d33b8b0..de0eaa3c49f 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -82,6 +82,11 @@ def mongod_program(  # pylint: disable=too-many-branches
     if "replSet" in kwargs and "writePeriodicNoops" not in suite_set_parameters:
         suite_set_parameters["writePeriodicNoops"] = False
 
+    # By default the primary waits up to 10 sec to complete a stepdown and to hand off its duties to
+    # a secondary before shutting down in response to SIGTERM. Make it shut down more abruptly.
+    if "replSet" in kwargs and "waitForStepDownOnNonCommandShutdown" not in suite_set_parameters:
+        suite_set_parameters["waitForStepDownOnNonCommandShutdown"] = False
+
     _apply_set_parameters(args, suite_set_parameters)
 
     shortcut_opts = {
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 8824c3f0940..61ea171ba04 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -192,6 +192,8 @@ namespace {
 const NamespaceString startupLogCollectionName("local.startup_log");
 const NamespaceString kSystemReplSetCollection("local.system.replset");
 
+MONGO_EXPORT_SERVER_PARAMETER(waitForStepDownOnNonCommandShutdown, bool, true);
+
 #ifdef _WIN32
 const ntservice::NtServiceDefaultStrings defaultServiceStrings = {
     L"MongoDB", L"MongoDB", L"MongoDB Server"};
@@ -889,8 +891,13 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) {
             }
 
             try {
+                // For faster tests, we allow a short wait time with setParameter.
+                auto waitTime = waitForStepDownOnNonCommandShutdown.load()
+                    ? Milliseconds(Seconds(10))
+                    : Milliseconds(100);
+
                 uassertStatusOK(
-                    replCoord->stepDown(opCtx, false /* force */, Seconds(10), Seconds(120)));
+                    replCoord->stepDown(opCtx, false /* force */, waitTime, Seconds(120)));
             } catch (const ExceptionFor<ErrorCodes::NotMaster>&) {
                 // ignore not master errors
             } catch (const DBException& e) {
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 3df3c69cc0e..b7e26396abb 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -2468,6 +2468,24 @@ var ReplSetTest = function(opts) {
             }
         }
 
+        // Make shutdown faster in tests, especially when election handoff has no viable candidate.
+        // Ignore errors from setParameter, perhaps mongod is too old for this parameter.
+        if (_callIsMaster()) {
+            asCluster(this._liveNodes, () => {
+                for (let node of this._liveNodes) {
+                    try {
+                        assert.commandWorked(node.adminCommand({
+                            setParameter: 1,
+                            waitForStepDownOnNonCommandShutdown: false,
+                        }));
+                    } catch (e) {
+                        print("Error in setParameter for waitForStepDownOnNonCommandShutdown:");
+                        print(e);
+                    }
+                }
+            });
+        }
+
         for (var i = 0; i < this.ports.length; i++) {
             this.stop(i, signal, opts);
         }