SERVER-37390 Run rollback test fixture with high election timeout to avoid any unplanned election.

(cherry picked from commit 02ce213b40c56096c9c57e093778b0889c335bb9)
(cherry picked from commit 5b85b8787d6e8cfd4234b09304f3538506f70bd9)
(cherry picked from commit 9b470eb73873f5db5c9fcee5df5316d477a1fa12)
(cherry picked from commit 04a2c9acc7ca061fb86736b377b897b11f6c7c48)
(cherry picked from commit f2aa1ffe05804aa3cc21ad5f980bca998dde09f3)
(cherry picked from commit 9aa9fa4a2844f0fe7890e01d621960a0c64607f6)

10 files changed, 206 insertions, 68 deletions

diff --git a/jstests/multiVersion/libs/multiversion_rollback.js b/jstests/multiVersion/libs/multiversion_rollback.js
index ef90e497c60..aad6fd25a58 100644
--- a/jstests/multiVersion/libs/multiversion_rollback.js
+++ b/jstests/multiVersion/libs/multiversion_rollback.js
@@ -110,7 +110,7 @@ function setupReplicaSet(testName, rollbackNodeVersion, syncSourceVersion) {
     // Start up a two-node cluster first.
     var rst = new ReplSetTest({name: testName, nodes: initialNodes, useBridge: true});
     rst.startSet();
-    rst.initiate();
+    rst.initiateWithHighElectionTimeout();
 
     // Wait for both nodes to be up.
     waitForState(rst.nodes[0], ReplSetTest.State.PRIMARY);
diff --git a/jstests/noPassthrough/rollback_wt_cache_full.js b/jstests/noPassthrough/rollback_wt_cache_full.js
index 6eb3a173db0..c67437fa4b4 100644
--- a/jstests/noPassthrough/rollback_wt_cache_full.js
+++ b/jstests/noPassthrough/rollback_wt_cache_full.js
@@ -25,7 +25,7 @@
         useBridge: true,
     });
     const nodes = rst.startSet();
-    rst.initiate();
+    rst.initiateWithHighElectionTimeout();
 
     // Prior to 4.0, rollback imposed a 300 MB limit on the total size of documents to refetch from
     // the sync source. Therefore, we select values for numDocs and minDocSizeMB, while accounting
diff --git a/jstests/replsets/avg_num_catchup_ops.js b/jstests/replsets/avg_num_catchup_ops.js
index 65418ff03d0..1f64d906070 100644
--- a/jstests/replsets/avg_num_catchup_ops.js
+++ b/jstests/replsets/avg_num_catchup_ops.js
@@ -11,14 +11,11 @@
     load("jstests/replsets/rslib.js");
 
     const name = jsTestName();
-    const rst = new ReplSetTest({name: name, nodes: 3, useBridge: true});
+    const rst = new ReplSetTest(
+        {name: name, nodes: 3, useBridge: true, settings: {catchUpTimeoutMillis: 4 * 60 * 1000}});
 
     rst.startSet();
-    const confSettings = {
-        catchUpTimeoutMillis: 4 * 60 * 1000,
-    };
-
-    rst.initiateWithHighElectionTimeout(confSettings);
+    rst.initiateWithHighElectionTimeout();
     rst.awaitSecondaryNodes();
     rst.awaitReplication();
 
diff --git a/jstests/replsets/libs/rollback_test.js b/jstests/replsets/libs/rollback_test.js
index 7c9a6f88088..957e4fe2adf 100644
--- a/jstests/replsets/libs/rollback_test.js
+++ b/jstests/replsets/libs/rollback_test.js
@@ -13,6 +13,28 @@
  * 4. kSyncSourceOpsDuringRollback: apply operations on the sync source after rollback has begun.
  * 5. kSteadyStateOps: (same as stage 1) with the option of waiting for the rollback to finish.
  *
+ * --------------------------------------------------
+ * | STATE TRANSITION            | NETWORK TOPOLOGY |
+ * |-------------------------------------------------
+ * |  kSteadyStateOps            |       A          |
+ * |                             |     /   \        |
+ * |                             |    P1 -  S       |
+ * |-----------------------------|------------------|
+ * |  kRollbackOps               |       A          |
+ * |                             |     /            |
+ * |                             |    P1    S       |
+ * |-----------------------------|------------------|
+ * | kSyncSourceOpsBeforeRollback|       A          |
+ * |                             |         \        |
+ * |                             |    P1    P2      |
+ * |-----------------------------|------------------|
+ * | kSyncSourceOpsDuringRollback|        A         |
+ * |                             |      /   \       |
+ * |                             |     R  -  P2     |
+ * |-------------------------------------------------
+ * Note: 'A' refers to arbiter node, 'S' refers to secondary, 'P[n]' refers to primary in
+ * nth term and 'R' refers to rollback node.
+ *
  * Please refer to the various `transition*` functions for more information on the behavior
  * of each stage.
  */
@@ -31,6 +53,8 @@ load("jstests/hooks/validate_collections.js");
  * requirements:
  *      1. It must have exactly three nodes: a primary, a secondary and an arbiter.
  *      2. It must be running with mongobridge.
+ *      3. Must initiate the replset with high election timeout to avoid unplanned elections in the
+ *         rollback test.
  *
  * If the caller does not provide their own replica set, a standard three-node
  * replset will be initialized instead, with all nodes running the latest version.
@@ -96,6 +120,13 @@ function RollbackTest(name = "RollbackTest", replSet) {
         arbiter = replSet.getArbiter();
         curSecondary = (secondaries[0] === arbiter) ? secondaries[1] : secondaries[0];
 
+        let config = replSet.getReplSetConfigFromNode();
+        // Make sure electionTimeoutMillis is set to high value to avoid unplanned elections in
+        // the rollback test.
+        assert.gte(config.settings.electionTimeoutMillis,
+                   ReplSetTest.kForeverMillis,
+                   "Must initiate the replset with high election timeout");
+
         waitForState(curSecondary, ReplSetTest.State.SECONDARY);
         waitForState(arbiter, ReplSetTest.State.ARBITER);
 
@@ -122,7 +153,7 @@ function RollbackTest(name = "RollbackTest", replSet) {
         replSet.startSet();
 
         const nodes = replSet.nodeList();
-        replSet.initiate({
+        replSet.initiateWithHighElectionTimeout({
             _id: name,
             members: [
                 {_id: 0, host: nodes[0]},
@@ -187,6 +218,21 @@ function RollbackTest(name = "RollbackTest", replSet) {
         }
     }
 
+    function stepUp(conn) {
+        log(`Waiting for the new primary ${conn.host} to be elected`);
+        assert.soonNoExcept(() => {
+            const res = conn.adminCommand({replSetStepUp: 1});
+            return res.ok;
+        });
+
+        // Waits for the primary to accept new writes.
+        return rst.getPrimary();
+    }
+
+    function oplogTop(conn) {
+        return conn.getDB("local").oplog.rs.find().limit(1).sort({$natural: -1}).next();
+    }
+
     /**
      * Transition from a rollback state to a steady state. Operations applied in this phase will
      * be replicated to all nodes and should not be rolled back.
@@ -197,49 +243,49 @@ function RollbackTest(name = "RollbackTest", replSet) {
         log(`Ensuring the secondary ${curSecondary.host} is connected to the other nodes`);
         curSecondary.reconnect([curPrimary, arbiter]);
 
-        // If we shut down the primary before the secondary begins rolling back against it, then
-        // the secondary may get elected and not actually roll back. In that case we do not
-        // check the RBID and just await replication.
-        if (!TestData.rollbackShutdowns) {
-            log(`Waiting for rollback to complete on ${curSecondary.host}`, true);
-            let rbid = -1;
-            assert.soon(() => {
-                try {
-                    rbid = assert.commandWorked(curSecondary.adminCommand("replSetGetRBID")).rbid;
-                } catch (e) {
-                    // Command can fail when sync source is being cleared.
+        // 1. Wait for the rollback node to be SECONDARY; this either waits for rollback to finish
+        // or exits early if it checks the node before it *enters* ROLLBACK.
+        //
+        // 2. Test that RBID is properly incremented; note that it could be incremented several
+        // times if the node restarts before a given rollback attempt finishes.
+        //
+        // 3. Check if the rollback node is caught up.
+        //
+        // If any conditions are unmet, retry.
+        //
+        // If {enableMajorityReadConcern:false} is set, it will use the rollbackViaRefetch
+        // algorithm. That can lead to unrecoverable rollbacks, particularly in unclean shutdown
+        // suites, as it is possible in rare cases for the sync source to lose the entry
+        // corresponding to the optime the rollback node chose as its minValid.
+
+        log(`Wait for ${curSecondary.host} to finish rollback`);
+        assert.soonNoExcept(() => {
+            try {
+                log(`Wait for secondary ${curSecondary}`);
+                rst.awaitSecondaryNodesForRollbackTest(
+                    awaitSecondaryNodesForRollbackTimeout,
+                    curSecondary /* connToCheckForUnrecoverableRollback */);
+            } catch (e) {
+                if (e.unrecoverableRollbackDetected) {
+                    log(`Detected unrecoverable rollback on ${curSecondary.host}. Ending test.`,
+                        true /* important */);
+                    TestData.skipCheckDBHashes = true;
+                    rst.stopSet();
+                    quit();
                 }
-                // Fail early if the rbid is greater than lastRBID+1.
-                assert.lte(rbid,
-                           lastRBID + 1,
-                           `RBID is too large. current RBID: ${rbid}, last RBID: ${lastRBID}`);
+                // Re-throw the original exception in all other cases.
+                throw e;
+            }
 
-                return rbid === lastRBID + 1;
-            }, "Timed out waiting for RBID to increment on " + curSecondary.host);
-        } else {
-            log(`Skipping RBID check on ${curSecondary.host} because shutdowns ` +
-                `may prevent a rollback here.`);
-        }
+            let rbid = assert.commandWorked(curSecondary.adminCommand("replSetGetRBID")).rbid;
+            assert(rbid > lastRBID,
+                   `Expected RBID to increment past ${lastRBID} on ${curSecondary.host}`);
+
+            assert.eq(oplogTop(curPrimary), oplogTop(curSecondary));
+
+            return true;
+        });
 
-        // If the rollback node has {enableMajorityReadConcern:false} set, it will use the
-        // rollbackViaRefetch algorithm. That can lead to unrecoverable rollbacks, particularly
-        // in unclean shutdown suites, as it it is possible in rare cases for the sync source to
-        // lose the entry corresponding to the optime the rollback node chose as its minValid.
-        try {
-            rst.awaitSecondaryNodesForRollbackTest(
-                awaitSecondaryNodesForRollbackTimeout,
-                curSecondary /* connToCheckForUnrecoverableRollback */);
-        } catch (e) {
-            if (e.unrecoverableRollbackDetected) {
-                log(`Detected unrecoverable rollback on ${curSecondary.host}. Ending test.`,
-                    true /* important */);
-                TestData.skipCheckDBHashes = true;
-                rst.stopSet();
-                quit();
-            }
-            // Re-throw the original exception in all other cases.
-            throw e;
-        }
         rst.awaitReplication();
 
         log(`Rollback on ${curSecondary.host} (if needed) and awaitReplication completed`, true);
@@ -313,13 +359,7 @@ function RollbackTest(name = "RollbackTest", replSet) {
         log(`Reconnecting the secondary ${curSecondary.host} to the arbiter so it can be elected`);
         curSecondary.reconnect([arbiter]);
 
-        log(`Waiting for the new primary ${curSecondary.host} to be elected`);
-        assert.soonNoExcept(() => {
-            const res = curSecondary.adminCommand({replSetStepUp: 1});
-            return res.ok;
-        });
-
-        const newPrimary = rst.getPrimary();
+        const newPrimary = stepUp(curSecondary);
 
         // As a sanity check, ensure the new primary is the old secondary. The opposite scenario
         // should never be possible with 2 electable nodes and the sequence of operations thus far.
@@ -331,6 +371,16 @@ function RollbackTest(name = "RollbackTest", replSet) {
         curSecondary = curPrimary;
         curPrimary = newPrimary;
 
+        // To ensure rollback won't be skipped for shutdowns, wait till the no-op oplog
+        // entry ("new primary") written in the new term gets persisted in the disk.
+        // Note: rollbackShutdowns are not allowed for in-memory/ephemeral storage engines.
+        if (TestData.rollbackShutdowns) {
+            const dbName = "TermGetsPersisted";
+            assert.commandWorked(curPrimary.getDB(dbName).ensureRollback.insert(
+                {thisDocument: 'is inserted to ensure rollback is not skipped'},
+                {writeConcern: {w: 1, j: true}}));
+        }
+
         lastRBID = assert.commandWorked(curSecondary.adminCommand("replSetGetRBID")).rbid;
 
         return curPrimary;
@@ -407,9 +457,34 @@ function RollbackTest(name = "RollbackTest", replSet) {
         log(`Restarting node ${hostName}`);
         rst.start(nodeId, startOptions, true /* restart */);
 
-        // Ensure that the primary is ready to take operations before continuing. If both nodes are
-        // connected to the arbiter, the primary may switch.
-        curPrimary = rst.getPrimary();
+        // Step up if the restarted node is the current primary.
+        if (rst.getNodeId(curPrimary) === nodeId) {
+            // To prevent below step up from being flaky, we step down and freeze the
+            // current secondary to prevent starting a new election. The current secondary
+            // can start running election due to explicit step up by the shutting down of current
+            // primary if the server parameter "enableElectionHandoff" is set to true.
+            rst.freeze(curSecondary);
+
+            const newPrimary = stepUp(curPrimary);
+            // As a sanity check, ensure the new primary is the current primary. This is true,
+            // because we have configured the replica set with high electionTimeoutMillis.
+            assert.eq(newPrimary, curPrimary, "Did not elect the same node as primary");
+
+            // Unfreeze the current secondary so that it can step up again. Retry on network errors
+            // in case the current secondary is in ROLLBACK state.
+            assert.soon(() => {
+                try {
+                    assert.commandWorked(curSecondary.adminCommand({replSetFreeze: 0}));
+                    return true;
+                } catch (e) {
+                    if (isNetworkError(e)) {
+                        return false;
+                    }
+                    throw e;
+                }
+            }, `Failed to unfreeze current secondary ${curSecondary.host}`);
+        }
+
         curSecondary = rst.getSecondary();
         assert.neq(curPrimary, curSecondary);
     };
diff --git a/jstests/replsets/rollback_after_disabling_majority_reads.js b/jstests/replsets/rollback_after_disabling_majority_reads.js
index 7b8ee2eb9ec..84eda89e7d8 100644
--- a/jstests/replsets/rollback_after_disabling_majority_reads.js
+++ b/jstests/replsets/rollback_after_disabling_majority_reads.js
@@ -18,7 +18,7 @@
     replTest.startSet();
     let config = replTest.getReplSetConfig();
     config.members[2].arbiterOnly = true;
-    replTest.initiate(config);
+    replTest.initiateWithHighElectionTimeout(config);
     const rollbackTest = new RollbackTest(name, replTest);
 
     const rollbackNode = rollbackTest.transitionToRollbackOperations();
diff --git a/jstests/replsets/rollback_after_enabling_majority_reads.js b/jstests/replsets/rollback_after_enabling_majority_reads.js
index f6d9c0aab1e..2b28be52e2a 100644
--- a/jstests/replsets/rollback_after_enabling_majority_reads.js
+++ b/jstests/replsets/rollback_after_enabling_majority_reads.js
@@ -23,7 +23,7 @@
     replTest.startSet();
     let config = replTest.getReplSetConfig();
     config.members[2].arbiterOnly = true;
-    replTest.initiate(config);
+    replTest.initiateWithHighElectionTimeout(config);
     let rollbackTest = new RollbackTest(name, replTest);
 
     let rollbackNode = rollbackTest.transitionToRollbackOperations();
diff --git a/jstests/replsets/rollback_via_refetch_survives_nonexistent_collection_drop.js b/jstests/replsets/rollback_via_refetch_survives_nonexistent_collection_drop.js
index 3dfce50504b..8218545bd93 100644
--- a/jstests/replsets/rollback_via_refetch_survives_nonexistent_collection_drop.js
+++ b/jstests/replsets/rollback_via_refetch_survives_nonexistent_collection_drop.js
@@ -27,7 +27,7 @@
 
     rst.startSet();
     const nodes = rst.nodeList();
-    rst.initiate({
+    rst.initiateWithHighElectionTimeout({
         _id: collName,
         members: [
             {_id: 0, host: nodes[0]},
diff --git a/jstests/replsets/transactions_after_rollback_via_refetch.js b/jstests/replsets/transactions_after_rollback_via_refetch.js
index 5d5ba119ecd..5ec265b5c2f 100644
--- a/jstests/replsets/transactions_after_rollback_via_refetch.js
+++ b/jstests/replsets/transactions_after_rollback_via_refetch.js
@@ -55,7 +55,7 @@
         {name, nodes: 3, useBridge: true, nodeOptions: {enableMajorityReadConcern: "false"}});
     replTest.startSet();
     const nodes = replTest.nodeList();
-    replTest.initiate({
+    replTest.initiateWithHighElectionTimeout({
         _id: name,
         members: [
             {_id: 0, host: nodes[0]},
diff --git a/jstests/replsets/unrecoverable_rollback_early_exit.js b/jstests/replsets/unrecoverable_rollback_early_exit.js
index d729b34dd0c..8dea3f88f1e 100644
--- a/jstests/replsets/unrecoverable_rollback_early_exit.js
+++ b/jstests/replsets/unrecoverable_rollback_early_exit.js
@@ -29,7 +29,7 @@
         nodeOptions: {enableMajorityReadConcern: "false"}
     });
     rst.startSet();
-    rst.initiate();
+    rst.initiateWithHighElectionTimeout();
 
     const rollbackTest = new RollbackTest(testName, rst);
     const rollbackNode = rollbackTest.transitionToRollbackOperations();
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 64e9ca7a770..a2cc63516e6 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -195,6 +195,40 @@ var ReplSetTest = function(opts) {
     }
 
     /**
+     * Wrap a function so it can accept a node id or connection as its first argument. The argument
+     * is converted to a connection.
+     */
+    function _nodeParamToConn(wrapped) {
+        return function(node, ...wrappedArgs) {
+            if (node.getDB) {
+                return wrapped.call(this, node, ...wrappedArgs);
+            }
+
+            assert(self.nodes.hasOwnProperty(node), `${node} not found in self.nodes`);
+            return wrapped.call(this, self.nodes[node], ...wrappedArgs);
+        };
+    }
+
+    /**
+     * Wrap a function so it accepts a single node or list of them as its first argument. The
+     * function is called once per node provided.
+     */
+    function _nodeParamToSingleNode(wrapped) {
+        return function(node, ...wrappedArgs) {
+            if (node.hasOwnProperty('length')) {
+                let returnValueList = [];
+                for (let i = 0; i < node.length; i++) {
+                    returnValueList.push(wrapped.call(this, node[i], ...wrappedArgs));
+                }
+
+                return returnValueList;
+            }
+
+            return wrapped.call(this, node, ...wrappedArgs);
+        };
+    }
+
+    /**
      * Wait for a rs indicator to go to a particular state or states.
      *
      * @param node is a single node or list of nodes, by id or conn
@@ -1167,10 +1201,11 @@ var ReplSetTest = function(opts) {
      * Modifies the election timeout to be 24 hours so that no unplanned elections happen. Then
      * runs replSetInitiate on the replica set with the new config.
      */
-    this.initiateWithHighElectionTimeout = function(opts = {}) {
-        let cfg = this.getReplSetConfig();
-        cfg.settings = Object.assign(opts, {"electionTimeoutMillis": 24 * 60 * 60 * 1000});
-        this.initiate(cfg);
+    this.initiateWithHighElectionTimeout = function(config) {
+        config = config || this.getReplSetConfig();
+        config.settings = config.settings || {};
+        config.settings["electionTimeoutMillis"] = ReplSetTest.kForeverMillis;
+        this.initiate(config);
     };
 
     /**
@@ -2505,6 +2540,31 @@ var ReplSetTest = function(opts) {
         return started;
     };
 
+    /**
+     * Step down and freeze a particular node or nodes.
+     *
+     * @param node is a single node or list of nodes, by id or conn
+     */
+    this.freeze = _nodeParamToSingleNode(_nodeParamToConn(function(node) {
+        assert.soon(() => {
+            try {
+                // Ensure node is not primary. Ignore errors, probably means it's already secondary.
+                node.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true});
+                // Prevent node from running election. Fails if it already started an election.
+                assert.commandWorked(node.adminCommand({replSetFreeze: ReplSetTest.kForeverSecs}));
+                return true;
+            } catch (e) {
+                if (isNetworkError(e) || e.code === ErrorCodes.NotSecondary ||
+                    e.code === ErrorCodes.NotYetInitialized) {
+                    jsTestLog(`Failed to freeze node ${node.host}: ${e}`);
+                    return false;
+                }
+
+                throw e;
+            }
+        }, `Failed to run replSetFreeze cmd on ${node.host}`);
+    }));
+
     this.stopMaster = function(signal, opts) {
         var master = this.getPrimary();
         var master_id = this.getNodeId(master);
@@ -2810,6 +2870,12 @@ var ReplSetTest = function(opts) {
 ReplSetTest.kDefaultTimeoutMS = 10 * 60 * 1000;
 
 /**
+ *  Global default number that's effectively infinite.
+ */
+ReplSetTest.kForeverSecs = 24 * 60 * 60;
+ReplSetTest.kForeverMillis = ReplSetTest.kForeverSecs * 1000;
+
+/**
  * Set of states that the replica set can be in. Used for the wait functions.
  */
 ReplSetTest.State = {