SERVER-27231 added disableMaxSyncSourceLagSecs failpoint to turn off re-evaluating sync sources, for jstests that manipulate chaining

(cherry picked from commit ae2f197bcb46ee961ebadbccba18154f94a09f56)

3 files changed, 50 insertions, 17 deletions

diff --git a/jstests/replsets/chaining_removal.js b/jstests/replsets/chaining_removal.js
index 269610ecae4..ee70a00289e 100644
--- a/jstests/replsets/chaining_removal.js
+++ b/jstests/replsets/chaining_removal.js
@@ -24,6 +24,16 @@
     var primary = replTest.getPrimary();
     replTest.awaitReplication();
 
+    // When setting up chaining on slow machines, we do not want slow writes or delayed heartbeats
+    // to cause our nodes to invalidate the sync source provided in the 'replSetSyncFrom' command.
+    // To achieve this, we disable the server parameter 'maxSyncSourceLagSecs' (see
+    // repl_settings_init.cpp and TopologyCoordinatorImpl::Options) in
+    // TopologyCoordinatorImpl::shouldChangeSyncSource().
+    assert.commandWorked(nodes[1].getDB('admin').runCommand(
+        {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'}));
+    assert.commandWorked(nodes[4].getDB('admin').runCommand(
+        {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'}));
+
     // Force node 1 to sync directly from node 0.
     assert.commandWorked(nodes[1].getDB("admin").runCommand({"replSetSyncFrom": nodes[0].host}));
     var res;
@@ -52,6 +62,12 @@
     var options = {writeConcern: {w: numNodes, wtimeout: timeout}};
     assert.writeOK(primary.getDB(name).foo.insert({x: 1}, options));
 
+    // Re-enable 'maxSyncSourceLagSecs' checking on sync source.
+    assert.commandWorked(nodes[1].getDB('admin').runCommand(
+        {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'}));
+    assert.commandWorked(nodes[4].getDB('admin').runCommand(
+        {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'}));
+
     var config = primary.getDB("local").system.replset.findOne();
     config.members.pop();
     config.version++;
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index 7832a1fe2b8..76c9d69e8eb 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -361,6 +361,7 @@ env.Library('topology_coordinator_impl',
                 'repl_settings',
                 'rslog',
                 'topology_coordinator',
+                '$BUILD_DIR/mongo/util/fail_point',
             ])
 
 env.CppUnitTest('repl_set_heartbeat_response_test',
diff --git a/src/mongo/db/repl/topology_coordinator_impl.cpp b/src/mongo/db/repl/topology_coordinator_impl.cpp
index 7e8405807b5..a96790c41a5 100644
--- a/src/mongo/db/repl/topology_coordinator_impl.cpp
+++ b/src/mongo/db/repl/topology_coordinator_impl.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/repl/rslog.h"
 #include "mongo/db/server_parameters.h"
 #include "mongo/rpc/metadata/repl_set_metadata.h"
+#include "mongo/util/fail_point_service.h"
 #include "mongo/util/hex.h"
 #include "mongo/util/log.h"
 #include "mongo/util/mongoutils/str.h"
@@ -65,6 +66,11 @@ const Seconds TopologyCoordinatorImpl::VoteLease::leaseTime = Seconds(30);
 // must be before it will call for a priority takeover election.
 MONGO_EXPORT_STARTUP_SERVER_PARAMETER(priorityTakeoverFreshnessWindowSeconds, int, 2);
 
+// If this fail point is enabled, TopologyCoordinatorImpl::shouldChangeSyncSource() will ignore
+// the option TopologyCoordinatorImpl::Options::maxSyncSourceLagSecs. The sync source will not be
+// re-evaluated if it lags behind another node by more than 'maxSyncSourceLagSecs' seconds.
+MONGO_FP_DECLARE(disableMaxSyncSourceLagSecs);
+
 namespace {
 
 template <typename T>
@@ -2472,26 +2478,36 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource(const HostAndPort& currentS
         return true;
     }
 
-    unsigned int currentSecs = currentSourceOpTime.getSecs();
-    unsigned int goalSecs = currentSecs + durationCount<Seconds>(_options.maxSyncSourceLagSecs);
+    if (MONGO_FAIL_POINT(disableMaxSyncSourceLagSecs)) {
+        log() << "disableMaxSyncSourceLagSecs fail point enabled - not checking the most recent "
+                 "OpTime, "
+              << currentSourceOpTime.toString() << ", of our current sync source, " << currentSource
+              << ", against the OpTimes of the other nodes in this replica set.";
+    } else {
+        unsigned int currentSecs = currentSourceOpTime.getSecs();
+        unsigned int goalSecs = currentSecs + durationCount<Seconds>(_options.maxSyncSourceLagSecs);
 
-    for (std::vector<MemberHeartbeatData>::const_iterator it = _hbdata.begin(); it != _hbdata.end();
-         ++it) {
-        const int itIndex = indexOfIterator(_hbdata, it);
-        const MemberConfig& candidateConfig = _rsConfig.getMemberAt(itIndex);
-        if (it->up() && (candidateConfig.isVoter() || !_selfConfig().isVoter()) &&
-            (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) &&
-            it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) &&
-            goalSecs < it->getAppliedOpTime().getSecs()) {
-            log() << "Choosing new sync source because the most recent OpTime of our sync source, "
-                  << currentSource << ", is " << currentSourceOpTime.toString()
-                  << " which is more than " << _options.maxSyncSourceLagSecs << " behind member "
-                  << candidateConfig.getHostAndPort().toString() << " whose most recent OpTime is "
-                  << it->getAppliedOpTime().toString();
-            invariant(itIndex != _selfIndex);
-            return true;
+        for (std::vector<MemberHeartbeatData>::const_iterator it = _hbdata.begin();
+             it != _hbdata.end();
+             ++it) {
+            const int itIndex = indexOfIterator(_hbdata, it);
+            const MemberConfig& candidateConfig = _rsConfig.getMemberAt(itIndex);
+            if (it->up() && (candidateConfig.isVoter() || !_selfConfig().isVoter()) &&
+                (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) &&
+                it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) &&
+                goalSecs < it->getAppliedOpTime().getSecs()) {
+                log() << "Choosing new sync source because the most recent OpTime of our sync "
+                         "source, "
+                      << currentSource << ", is " << currentSourceOpTime.toString()
+                      << " which is more than " << _options.maxSyncSourceLagSecs
+                      << " behind member " << candidateConfig.getHostAndPort().toString()
+                      << " whose most recent OpTime is " << it->getAppliedOpTime().toString();
+                invariant(itIndex != _selfIndex);
+                return true;
+            }
         }
     }
+
     return false;
 }