SERVER-26772 removed old initial sync code

10 files changed, 323 insertions, 135 deletions

diff --git a/jstests/replsets/initial_sync_applier_error.js b/jstests/replsets/initial_sync_applier_error.js
index 543e26d49b7..9816e92fd74 100644
--- a/jstests/replsets/initial_sync_applier_error.js
+++ b/jstests/replsets/initial_sync_applier_error.js
@@ -13,12 +13,6 @@
     "use strict";
     load("jstests/libs/check_log.js");
 
-    var parameters = TestData.setParameters;
-    if (parameters && parameters.indexOf("use3dot2InitialSync=true") != -1) {
-        jsTest.log("Skipping this test because use3dot2InitialSync was provided.");
-        return;
-    }
-
     var name = 'initial_sync_applier_error';
     var replSet = new ReplSetTest({
         name: name,
diff --git a/jstests/replsets/initial_sync_cloner_dups.js b/jstests/replsets/initial_sync_cloner_dups.js
new file mode 100644
index 00000000000..23b1b989400
--- /dev/null
+++ b/jstests/replsets/initial_sync_cloner_dups.js
@@ -0,0 +1,132 @@
+/**
+ * Test for SERVER-17487
+ * 3 node replset
+ * insert docs with numeric _ids
+ * start deleting/re-inserting docs from collection in a loop
+ * add new secondary to force initialSync
+ * verify collection and both indexes on the secondary have the right number of docs
+ */
+(function(doNotRun) {
+    "use strict";
+
+    if (doNotRun) {
+        return;
+    }
+
+    load('jstests/libs/parallelTester.js');
+
+    Random.setRandomSeed();
+
+    // used to parse RAM log file
+    var contains = function(logLines, func) {
+        var i = logLines.length;
+        while (i--) {
+            printjson(logLines[i]);
+            if (func(logLines[i])) {
+                return true;
+            }
+        }
+        return false;
+    };
+
+    var replTest = new ReplSetTest({name: 'cloner', nodes: 3, oplogSize: 150 /*~1.5x data size*/});
+    replTest.startSet();
+    var conf = replTest.getReplSetConfig();
+    conf.settings = {};
+    conf.settings.chainingAllowed = false;
+    replTest.initiate(conf);
+    replTest.awaitSecondaryNodes();
+    var primary = replTest.getPrimary();
+    var coll = primary.getDB('test').cloner;
+    coll.drop();
+    coll.createIndex({k: 1});
+
+    // These need to be big enough to force initial-sync to use many batches
+    var numDocs = 100 * 1000;
+    var bigStr = Array(1001).toString();
+    var batch = coll.initializeUnorderedBulkOp();
+    for (var i = 0; i < numDocs; i++) {
+        batch.insert({_id: i, bigStr: bigStr});
+    }
+    batch.execute();
+
+    replTest.awaitReplication();
+
+    jsTestLog("Start remove/insert on primary");
+    var insertAndRemove = function(host) {
+        jsTestLog("starting bg writes on " + host);
+        var m = new Mongo(host);
+        var db = m.getDB('test');
+        var coll = db.cloner;
+        var numDocs = coll.count();
+        for (var i = 0; !db.stop.findOne(); i++) {
+            var id = Random.randInt(numDocs);
+            coll.remove({_id: id});
+            coll.insert({_id: id});
+
+            var id = i % numDocs;
+            // print(id);
+            coll.remove({_id: id});
+            coll.insert({_id: id});
+
+            // Try to throttle this thread to prevent overloading slow machines.
+            sleep(1);
+        }
+
+        jsTestLog("finished bg writes on " + host);
+    };
+    var worker = new ScopedThread(insertAndRemove, primary.host);
+    worker.start();
+
+    jsTestLog("add a new secondary");
+    var secondary = replTest.add({});
+    replTest.reInitiate();
+    secondary.setSlaveOk();
+    // Wait for the secondary to get ReplSetInitiate command.
+    replTest.waitForState(
+        secondary,
+        [ReplSetTest.State.STARTUP_2, ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY]);
+
+    // This fail point will cause the first intial sync to fail, and leave an op in the buffer to
+    // verify the fix from SERVER-17807
+    print("=================== failpoint enabled ==============");
+    printjson(assert.commandWorked(secondary.getDB("admin").adminCommand(
+        {configureFailPoint: 'failInitSyncWithBufferedEntriesLeft', mode: {times: 1}})));
+    printjson(assert.commandWorked(secondary.getDB("admin").adminCommand({resync: true})));
+
+    // NOTE: This is here to prevent false negatives, but it is racy and dependent on magic numbers.
+    // Removed the assertion because it was too flaky.  Printing a warning instead (dan)
+    jsTestLog("making sure we dropped some dups");
+    var res = secondary.adminCommand({getLog: "global"});
+    var droppedDups = (contains(res.log, function(v) {
+        return v.indexOf("index build dropped" /* NNN dups*/) != -1;
+    }));
+    if (!droppedDups) {
+        jsTestLog(
+            "Warning: Test did not trigger duplicate documents, this run will be a false negative");
+    }
+
+    jsTestLog("stopping writes and waiting for replica set to coalesce");
+    primary.getDB('test').stop.insert({});
+    worker.join();
+    // make sure all secondaries are caught up, after init sync
+    reconnect(secondary.getDB("test"));
+    replTest.awaitSecondaryNodes();
+    replTest.awaitReplication();
+
+    jsTestLog("check that secondary has correct counts");
+    var secondaryColl = secondary.getDB('test').getCollection('cloner');
+    var index = secondaryColl.find({}, {_id: 1}).hint({_id: 1}).itcount();
+    var secondary_index = secondaryColl.find({}, {_id: 1}).hint({k: 1}).itcount();
+    var table = secondaryColl.find({}, {_id: 1}).hint({$natural: 1}).itcount();
+    if (index != table || index != secondary_index) {
+        printjson({
+            name: coll,
+            _id_index_count: index,
+            secondary_index_count: secondary_index,
+            table_count: table
+        });
+    }
+    assert.eq(index, table);
+    assert.eq(table, secondary_index);
+})(true /* Disabled until SERVER-23476 re-enabled rsync command */);
diff --git a/jstests/replsets/initial_sync_invalid_index_spec.js b/jstests/replsets/initial_sync_invalid_index_spec.js
index a40f6f14a79..ce608334e8b 100644
--- a/jstests/replsets/initial_sync_invalid_index_spec.js
+++ b/jstests/replsets/initial_sync_invalid_index_spec.js
@@ -30,16 +30,9 @@
     const msgInvalidOption = "The field 'invalidOption' is not valid for an index specification";
     const msgInitialSyncFatalAssertion = "Fatal assertion 40088 InitialSyncFailure";
 
-    // As part of the initsync-3dot2-rhel-62 evergreen variant, we run this test with a setParameter
-    // of "use3dot2InitialSync=true" which exercises 3.2 initial sync behavior. This path will
-    // trigger a different fatal assertion than the normal 3.4 path, which we need to handle here.
-    // TODO: Remove this assertion check when the 'use3dot2InitialSync' setParameter is retired.
-    const msg3dot2InitialSyncFatalAssertion = "Fatal Assertion 16233";
-
     const assertFn = function() {
         return rawMongoProgramOutput().match(msgInvalidOption) &&
-            (rawMongoProgramOutput().match(msgInitialSyncFatalAssertion) ||
-             rawMongoProgramOutput().match(msg3dot2InitialSyncFatalAssertion));
+            rawMongoProgramOutput().match(msgInitialSyncFatalAssertion);
     };
     assert.soon(assertFn, "Initial sync should have aborted on invalid index specification");
 
diff --git a/jstests/replsets/initial_sync_invalid_views.js b/jstests/replsets/initial_sync_invalid_views.js
new file mode 100644
index 00000000000..7b6b76bf31d
--- /dev/null
+++ b/jstests/replsets/initial_sync_invalid_views.js
@@ -0,0 +1,34 @@
+// Previously, the listCollections command would validate the views in all cases, potentially
+// causing a secondary to crash in the initial sync of a replicate set in the case that invalid
+// views were present. This test ensures that crashes no longer occur in those circumstances.
+
+(function() {
+    'use strict';
+
+    const name = "initial_sync_invalid_views";
+    let replSet = new ReplSetTest({name: name, nodes: 1});
+
+    let oplogSizeOnPrimary = 1;  // size in MB
+    replSet.startSet({oplogSize: oplogSizeOnPrimary});
+    replSet.initiate();
+    let primary = replSet.getPrimary();
+
+    let coll = primary.getDB('test').foo;
+    assert.writeOK(coll.insert({a: 1}));
+
+    // Add a secondary node but make it hang before copying databases.
+    let secondary = replSet.add();
+    secondary.setSlaveOk();
+
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'alwaysOn'}));
+    replSet.reInitiate();
+
+    assert.writeOK(primary.getDB('test').system.views.insert({invalid: NumberLong(1000)}));
+
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'}));
+
+    replSet.awaitSecondaryNodes(200 * 1000);
+    replSet.stopSet();
+})();
diff --git a/jstests/replsets/initial_sync_move_forward.js b/jstests/replsets/initial_sync_move_forward.js
index 6b01a32cbfd..410b1eb707a 100644
--- a/jstests/replsets/initial_sync_move_forward.js
+++ b/jstests/replsets/initial_sync_move_forward.js
@@ -38,7 +38,7 @@
     assert.commandWorked(masterColl.ensureIndex({x: 1}, {unique: true}));
 
     // Add a secondary.
-    var secondary = rst.add({setParameter: "num3Dot2InitialSyncAttempts=1"});
+    var secondary = rst.add();
     secondary.setSlaveOk();
     var secondaryColl = secondary.getDB("test").coll;
 
diff --git a/jstests/replsets/initial_sync_oplog_rollover.js b/jstests/replsets/initial_sync_oplog_rollover.js
new file mode 100644
index 00000000000..858b9d279d3
--- /dev/null
+++ b/jstests/replsets/initial_sync_oplog_rollover.js
@@ -0,0 +1,65 @@
+/**
+ * This test tests that initial sync succeeds when the sync source's oplog rolls over before the
+ * destination node reaches the oplog apply phase. It adds a new secondary to a replicaset and then
+ * pauses the initial sync before it copies the databases but after it starts to fetch and buffer
+ * oplog entries. The primary then fills up its oplog until it rolls over. At that point
+ * initial sync is resumed and we assert that it succeeds and that all of the inserted documents
+ * are there.
+ */
+
+(function() {
+    "use strict";
+    load("jstests/libs/check_log.js");
+
+    var name = 'initial_sync_oplog_rollover';
+    var replSet = new ReplSetTest({
+        name: name,
+        nodes: 1,
+    });
+
+    var oplogSizeOnPrimary = 1;  // size in MB
+    replSet.startSet({oplogSize: oplogSizeOnPrimary});
+    replSet.initiate();
+    var primary = replSet.getPrimary();
+
+    var coll = primary.getDB('test').foo;
+    assert.writeOK(coll.insert({a: 1}));
+
+    function getFirstOplogEntry(conn) {
+        return conn.getDB('local').oplog.rs.find().sort({ts: 1}).limit(1)[0];
+    }
+
+    var firstOplogEntry = getFirstOplogEntry(primary);
+
+    // Add a secondary node but make it hang before copying databases.
+    var secondary = replSet.add();
+    secondary.setSlaveOk();
+
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'alwaysOn'}));
+    replSet.reInitiate();
+
+    checkLog.contains(secondary,
+                      'initial sync - initialSyncHangBeforeCopyingDatabases fail point enabled');
+
+    // Keep inserting large documents until they roll over the oplog.
+    const largeStr = new Array(4 * 1024 * oplogSizeOnPrimary).join('aaaaaaaa');
+    var i = 0;
+    while (bsonWoCompare(getFirstOplogEntry(primary), firstOplogEntry) === 0) {
+        assert.writeOK(coll.insert({a: 2, x: i++, long_str: largeStr}));
+        sleep(100);
+    }
+
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'}));
+
+    replSet.awaitSecondaryNodes(200 * 1000);
+
+    assert.eq(i,
+              secondary.getDB('test').foo.count({a: 2}),
+              'collection successfully synced to secondary');
+
+    assert.eq(0,
+              secondary.getDB('local')['temp_oplog_buffer'].find().itcount(),
+              "Oplog buffer was not dropped after initial sync");
+})();
diff --git a/jstests/replsets/initial_sync_replSetGetStatus.js b/jstests/replsets/initial_sync_replSetGetStatus.js
new file mode 100644
index 00000000000..265a393374f
--- /dev/null
+++ b/jstests/replsets/initial_sync_replSetGetStatus.js
@@ -0,0 +1,90 @@
+/**
+ * This test tests that replSetGetStatus returns initial sync stats while initial sync is in
+ * progress.
+ */
+
+(function() {
+    "use strict";
+    load("jstests/libs/check_log.js");
+
+    var name = 'initial_sync_replSetGetStatus';
+    var replSet = new ReplSetTest({
+        name: name,
+        nodes: 1,
+    });
+
+    replSet.startSet();
+    replSet.initiate();
+    var primary = replSet.getPrimary();
+
+    var coll = primary.getDB('test').foo;
+    assert.writeOK(coll.insert({a: 1}));
+    assert.writeOK(coll.insert({a: 2}));
+
+    // Add a secondary node but make it hang before copying databases.
+    var secondary = replSet.add();
+    secondary.setSlaveOk();
+
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'alwaysOn'}));
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeFinish', mode: 'alwaysOn'}));
+    replSet.reInitiate();
+
+    // Wait for initial sync to pause before it copies the databases.
+    checkLog.contains(secondary,
+                      'initial sync - initialSyncHangBeforeCopyingDatabases fail point enabled');
+
+    // Test that replSetGetStatus returns the correct results while initial sync is in progress.
+    var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
+    assert(!res.initialSyncStatus,
+           "Response should not have an 'initialSyncStatus' field: " + tojson(res));
+
+    res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    assert(res.initialSyncStatus,
+           "Response should have an 'initialSyncStatus' field: " + tojson(res));
+
+    assert.commandFailed(secondary.adminCommand({replSetGetStatus: 1, initialSync: "t"}),
+                         ErrorCodes.TypeMismatch);
+
+    assert.writeOK(coll.insert({a: 3}));
+    assert.writeOK(coll.insert({a: 4}));
+
+    // Let initial sync continue working.
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'}));
+
+    // Wait for initial sync to pause right before it finishes.
+    checkLog.contains(secondary, 'initial sync - initialSyncHangBeforeFinish fail point enabled');
+
+    // Test that replSetGetStatus returns the correct results when initial sync is at the very end.
+    res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    assert(res.initialSyncStatus, "Response should have an 'initialSyncStatus' field.");
+    assert.eq(res.initialSyncStatus.fetchedMissingDocs, 0);
+    assert.eq(res.initialSyncStatus.appliedOps, 2);
+    assert.eq(res.initialSyncStatus.failedInitialSyncAttempts, 0);
+    assert.eq(res.initialSyncStatus.maxFailedInitialSyncAttempts, 1);
+    assert.eq(res.initialSyncStatus.databases.databasesCloned, 2);
+    assert.eq(res.initialSyncStatus.databases.test.collections, 1);
+    assert.eq(res.initialSyncStatus.databases.test.clonedCollections, 1);
+    assert.eq(res.initialSyncStatus.databases.test["test.foo"].documentsToCopy, 4);
+    assert.eq(res.initialSyncStatus.databases.test["test.foo"].documentsCopied, 4);
+    assert.eq(res.initialSyncStatus.databases.test["test.foo"].indexes, 1);
+    assert.eq(res.initialSyncStatus.databases.test["test.foo"].fetchedBatches, 1);
+
+    // Let initial sync finish and get into secondary state.
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'initialSyncHangBeforeFinish', mode: 'off'}));
+    replSet.awaitSecondaryNodes(60 * 1000);
+
+    // Test that replSetGetStatus returns the correct results after initial sync is finished.
+    res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
+    assert(!res.initialSyncStatus,
+           "Response should not have an 'initialSyncStatus' field: " + tojson(res));
+
+    assert.commandFailedWithCode(secondary.adminCommand({replSetGetStatus: 1, initialSync: "m"}),
+                                 ErrorCodes.TypeMismatch);
+    assert.eq(0,
+              secondary.getDB('local')['temp_oplog_buffer'].find().itcount(),
+              "Oplog buffer was not dropped after initial sync");
+})();
diff --git a/jstests/replsets/initial_sync_update_missing_doc1.js b/jstests/replsets/initial_sync_update_missing_doc1.js
index d2d3c6aed58..205d080da2a 100644
--- a/jstests/replsets/initial_sync_update_missing_doc1.js
+++ b/jstests/replsets/initial_sync_update_missing_doc1.js
@@ -13,12 +13,6 @@
 (function() {
     load("jstests/libs/check_log.js");
 
-    var parameters = TestData.setParameters;
-    if (parameters && parameters.indexOf("use3dot2InitialSync=true") != -1) {
-        jsTest.log("Skipping this test because use3dot2InitialSync was provided.");
-        return;
-    }
-
     var name = 'initial_sync_update_missing_doc1';
     var replSet = new ReplSetTest({
         name: name,
diff --git a/jstests/replsets/initial_sync_update_missing_doc2.js b/jstests/replsets/initial_sync_update_missing_doc2.js
index 9418757468c..c00ceb7c638 100644
--- a/jstests/replsets/initial_sync_update_missing_doc2.js
+++ b/jstests/replsets/initial_sync_update_missing_doc2.js
@@ -13,12 +13,6 @@
 (function() {
     load("jstests/libs/check_log.js");
 
-    var parameters = TestData.setParameters;
-    if (parameters && parameters.indexOf("use3dot2InitialSync=true") != -1) {
-        jsTest.log("Skipping this test because use3dot2InitialSync was provided.");
-        return;
-    }
-
     var name = 'initial_sync_update_missing_doc2';
     var replSet = new ReplSetTest({
         name: name,
diff --git a/jstests/replsets/two_initsync.js b/jstests/replsets/two_initsync.js
deleted file mode 100644
index 2850f756912..00000000000
--- a/jstests/replsets/two_initsync.js
+++ /dev/null
@@ -1,108 +0,0 @@
-// test initial sync failing
-
-// try running as :
-//
-//   mongo --nodb two_initsync.js | tee out | grep -v ^m31
-//
-
-var debugging = 0;
-
-function pause(s) {
-    print(s);
-    while (debugging) {
-        sleep(3000);
-        print(s);
-    }
-}
-
-function deb(obj) {
-    if (debugging) {
-        print("\n\n\n" + obj + "\n\n");
-    }
-}
-
-w = 0;
-
-function wait(f) {
-    w++;
-    var n = 0;
-    while (!f()) {
-        if (n % 4 == 0)
-            print("twoinitsync waiting " + w);
-        if (++n == 4) {
-            print("" + f);
-        }
-        assert(n < 200, 'tried 200 times, giving up');
-        sleep(1000);
-    }
-}
-
-doTest = function(signal) {
-    if (!(typeof TestData.setParameters === 'string') ||
-        !TestData.setParameters.includes("use3dot2InitialSync=true")) {
-        print("Test should only run with 3.2 style initial sync.");
-        return;
-    }
-    var replTest = new ReplSetTest({name: 'testSet', nodes: 0});
-
-    var first = replTest.add();
-
-    // Initiate replica set
-    assert.soon(function() {
-        var res = first.getDB("admin").runCommand({replSetInitiate: null});
-        return res['ok'] == 1;
-    });
-
-    // Get status
-    assert.soon(function() {
-        var result = first.getDB("admin").runCommand({replSetGetStatus: true});
-        return result['ok'] == 1;
-    });
-
-    var a = replTest.getPrimary().getDB("two");
-    for (var i = 0; i < 20000; i++)
-        assert.writeOK(a.coll.insert({
-            i: i,
-            s: "a                                                                       b"
-        }));
-    assert.eq(20000, a.coll.find().itcount());
-
-    // Start a second node
-    var second = replTest.add();
-
-    // Add the second node.
-    // This runs the equivalent of rs.add(newNode);
-    replTest.reInitiate(60000);
-
-    var b = second.getDB("admin");
-
-    // attempt to interfere with the initial sync
-    b._adminCommand({replSetTest: 1, forceInitialSyncFailure: 1});
-
-    //    wait(function () { return a._adminCommand("replSetGetStatus").members.length == 2; });
-
-    wait(function() {
-        return b.isMaster().secondary || b.isMaster().ismaster;
-    });
-
-    print("b.isMaster:");
-    printjson(b.isMaster());
-
-    second.setSlaveOk();
-
-    print("b.isMaster:");
-    printjson(b.isMaster());
-
-    wait(function() {
-        var c = b.getSisterDB("two").coll.find().itcount();
-        print(c);
-        return c == 20000;
-    });
-
-    print("two_initsync.js SUCCESS");
-
-    replTest.stopSet(signal);
-};
-
-print("two_initsync.js");
-doTest(15);