SERVER-27071 ensure config.version doc cannot replicate to secondaries in config_version_rollback.js

2 files changed, 100 insertions, 14 deletions

diff --git a/jstests/sharding/config_version_rollback.js b/jstests/sharding/config_version_rollback.js
index 5c0b9af2a8b..2fc5d78c8fc 100644
--- a/jstests/sharding/config_version_rollback.js
+++ b/jstests/sharding/config_version_rollback.js
@@ -7,27 +7,96 @@
 (function() {
     "use strict";
 
+    // Wait for fail point message to be logged.
+    var checkLog = function(node, msg) {
+        assert.soon(
+            function() {
+                var logMessages = assert.commandWorked(node.adminCommand({getLog: 'global'})).log;
+                for (var i = 0; i < logMessages.length; i++) {
+                    if (logMessages[i].indexOf(msg) != -1) {
+                        return true;
+                    }
+                }
+                return false;
+            },
+            'Did not see a log entry for ' + node + ' containing the following message: ' + msg,
+            60000,
+            1000);
+    };
+
+    // The config.version document is written on transition to primary. We need to ensure this
+    // config.version document is rolled back for this test.
+    //
+    // This means we have to guarantee the config.version document is not replicated by a secondary
+    // during any of 1) initial sync, 2) steady state replication, or 3) catchup after election.
+    //
+    // 1) initial sync
+    // We need non-primaries to finish initial sync so that they are electable, but without
+    // replicating the config.version document. Since we can't control when the config.version
+    // document is written (it's an internal write, not a client write), we turn on a failpoint
+    // that stalls the write of the config.version document until we have ascertained that the
+    // secondaries have finished initial sync.
+    //
+    // 2) steady state replication
+    // Once the non-primaries have transitioned to secondary, we stop the secondaries from
+    // replicating anything further by turning on a failpoint that stops the OplogFetcher. We then
+    // allow the primary to write the config.verison document.
+    //
+    // 3) catchup after election
+    // When the primary is stepped down and one of the secondaries is elected, the new primary will
+    // notice that it is behind the original primary and try to catchup for a short period. So, we
+    // also ensure that this short period is 0 by setting catchupTimeoutMillis to 0 earlier in the
+    // ReplSetConfig passed to initiate().
+    //
+    // Thus, we guarantee the new primary will not have replicated the config.version document in
+    // initial sync, steady state replication, or catchup, so the document will be rolled back.
+
+    jsTest.log("Starting the replica set and waiting for secondaries to finish initial sync");
     var configRS = new ReplSetTest({nodes: 3});
-    var nodes = configRS.startSet({configsvr: '', storageEngine: 'wiredTiger'});
+    var nodes = configRS.startSet({
+        configsvr: '',
+        storageEngine: 'wiredTiger',
+        setParameter: {
+            "failpoint.transitionToPrimaryHangBeforeInitializingConfigDatabase":
+                "{'mode':'alwaysOn'}"
+        }
+    });
+    var conf = configRS.getReplSetConfig();
+    conf.settings = {catchUpTimeoutMillis: 0};
+    configRS.initiate(conf);
 
-    // Prevent any replication from happening, so that the initial writes that the config
-    // server performs on first transition to primary can be rolled back.
-    //
-    // We cannot stop bgsync here because the new primary needs it to complete drain mode,
-    // so we let the bgsync on secondaries keep trying but fail to sync anything new.
-    nodes.forEach(function(node) {
+    var secondaries = configRS.getSecondaries();
+    var origPriConn = configRS.getPrimary();
+
+    // Ensure the primary is waiting to write the config.version document before stopping the oplog
+    // fetcher on the secondaries.
+    checkLog(
+        origPriConn,
+        'transition to primary - transitionToPrimaryHangBeforeInitializingConfigDatabase fail point enabled.');
+
+    jsTest.log("Stopping the OplogFetcher on the secondaries");
+    secondaries.forEach(function(node) {
         assert.commandWorked(node.getDB('admin').runCommand(
             {configureFailPoint: 'stopOplogFetcher', mode: 'alwaysOn'}));
     });
 
-    configRS.initiate();
-
-    var origPriConn = configRS.getPrimary();
-    var secondaries = configRS.getSecondaries();
+    jsTest.log("Allowing the primary to write the config.version doc");
+    // Note: since we didn't know which node would be elected to be the first primary, we had to
+    // turn this failpoint on for all nodes earlier. Since we do want the all future primaries to
+    // write the config.version doc immediately, we turn the failpoint off for all nodes now.
+    nodes.forEach(function(node) {
+        assert.commandWorked(node.adminCommand({
+            configureFailPoint: "transitionToPrimaryHangBeforeInitializingConfigDatabase",
+            mode: "off"
+        }));
+    });
 
     jsTest.log("Confirming that the primary has the config.version doc but the secondaries do not");
-    var origConfigVersionDoc = origPriConn.getCollection('config.version').findOne();
-    assert.neq(null, origConfigVersionDoc);
+    var origConfigVersionDoc;
+    assert.soon(function() {
+        origConfigVersionDoc = origPriConn.getCollection('config.version').findOne();
+        return null !== origConfigVersionDoc;
+    });
     secondaries.forEach(function(secondary) {
         secondary.setSlaveOk();
         assert.eq(null, secondary.getCollection('config.version').findOne());
@@ -53,13 +122,14 @@
     assert.neq(origConfigVersionDoc.clusterId, newConfigVersionDoc.clusterId);
 
     jsTest.log("Re-enabling replication on all nodes");
-    nodes.forEach(function(node) {
+    secondaries.forEach(function(node) {
         assert.commandWorked(
             node.getDB('admin').runCommand({configureFailPoint: 'stopOplogFetcher', mode: 'off'}));
     });
 
     jsTest.log(
         "Waiting for original primary to rollback and replicate new config.version document");
+    configRS.waitForState(origPriConn, ReplSetTest.State.SECONDARY);
     origPriConn.setSlaveOk();
     assert.soonNoExcept(function() {
         var foundClusterId = origPriConn.getCollection('config.version').findOne().clusterId;
@@ -110,4 +180,5 @@
               shardIdentityDoc.clusterId,
               "oldPriClusterId: " + origConfigVersionDoc.clusterId);
     configRS.stopSet();
+
 })();
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
index 9413d1219af..afee6b33dea 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
@@ -87,6 +87,7 @@
 #include "mongo/util/assert_util.h"
 #include "mongo/util/concurrency/thread_pool.h"
 #include "mongo/util/exit.h"
+#include "mongo/util/fail_point_service.h"
 #include "mongo/util/log.h"
 #include "mongo/util/mongoutils/str.h"
 #include "mongo/util/net/hostandport.h"
@@ -96,6 +97,8 @@
 namespace mongo {
 namespace repl {
 
+MONGO_FP_DECLARE(transitionToPrimaryHangBeforeInitializingConfigDatabase);
+
 namespace {
 using UniqueLock = stdx::unique_lock<stdx::mutex>;
 using LockGuard = stdx::lock_guard<stdx::mutex>;
@@ -726,6 +729,18 @@ void ReplicationCoordinatorExternalStateImpl::_shardingOnTransitionToPrimaryHook
     fassertStatusOK(40107, status);
 
     if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) {
+        if (MONGO_FAIL_POINT(transitionToPrimaryHangBeforeInitializingConfigDatabase)) {
+            log() << "transition to primary - "
+                     "transitionToPrimaryHangBeforeInitializingConfigDatabase fail point enabled. "
+                     "Blocking until fail point is disabled.";
+            while (MONGO_FAIL_POINT(transitionToPrimaryHangBeforeInitializingConfigDatabase)) {
+                mongo::sleepsecs(1);
+                if (inShutdown()) {
+                    break;
+                }
+            }
+        }
+
         status = Grid::get(txn)->catalogManager()->initializeConfigDatabaseIfNeeded(txn);
         if (!status.isOK() && status != ErrorCodes::AlreadyInitialized) {
             if (ErrorCodes::isShutdownError(status.code())) {