SERVER-35731 Prevent a repaired node from re-joining a replica set

4 files changed, 323 insertions, 2 deletions

diff --git a/jstests/disk/libs/wt_file_helper.js b/jstests/disk/libs/wt_file_helper.js
index f8483e0047f..259b9a04bcd 100644
--- a/jstests/disk/libs/wt_file_helper.js
+++ b/jstests/disk/libs/wt_file_helper.js
@@ -24,6 +24,95 @@ let corruptFile = function(file) {
 };
 
 /**
+ * Assert that running MongoDB with --repair on the provided dbpath exits cleanly.
+ */
+let assertRepairSucceeds = function(dbpath, port) {
+    jsTestLog("Repairing the node");
+    assert.eq(0, runMongoProgram("mongod", "--repair", "--port", port, "--dbpath", dbpath));
+};
+
+let assertRepairFailsWithFailpoint = function(dbpath, port, failpoint) {
+    const param = "failpoint." + failpoint + "={'mode': 'alwaysOn'}";
+    jsTestLog("The node should fail to complete repair with --setParameter " + param);
+
+    assert.eq(
+        MongoRunner.EXIT_ABRUPT,
+        runMongoProgram(
+            "mongod", "--repair", "--port", port, "--dbpath", dbpath, "--setParameter", param));
+};
+
+/**
+ * Assert that starting MongoDB with --replSet on an existing data path exits with a specific
+ * error.
+ */
+let assertErrorOnStartupWhenStartingAsReplSet = function(dbpath, port, rsName) {
+    jsTestLog("The repaired node should fail to start up with the --replSet option");
+
+    clearRawMongoProgramOutput();
+    let node = MongoRunner.runMongod(
+        {dbpath: dbpath, port: port, replSet: rsName, noCleanData: true, waitForConnect: false});
+    assert.soon(function() {
+        return rawMongoProgramOutput().indexOf("Fatal Assertion 50923") >= 0;
+    });
+    MongoRunner.stopMongod(node, null, {allowedExitCode: MongoRunner.EXIT_ABRUPT});
+};
+
+/**
+ * Assert that starting MongoDB as a standalone on an existing data path exits with a specific
+ * error because the previous repair failed.
+ */
+let assertErrorOnStartupAfterIncompleteRepair = function(dbpath, port) {
+    jsTestLog("The node should fail to start up because a previous repair did not complete");
+
+    clearRawMongoProgramOutput();
+    let node = MongoRunner.runMongod(
+        {dbpath: dbpath, port: port, noCleanData: true, waitForConnect: false});
+    assert.soon(function() {
+        return rawMongoProgramOutput().indexOf("Fatal Assertion 50922") >= 0;
+    });
+    MongoRunner.stopMongod(node, null, {allowedExitCode: MongoRunner.EXIT_ABRUPT});
+};
+
+/**
+ * Assert that starting MongoDB as a standalone on an existing data path succeeds. Uses a provided
+ * testFunc to run any caller-provided checks on the started node.
+ */
+let assertStartAndStopStandaloneOnExistingDbpath = function(dbpath, port, testFunc) {
+    jsTestLog("The repaired node should start up and serve reads as a standalone");
+    let node = MongoRunner.runMongod({dbpath: dbpath, port: port, noCleanData: true});
+    assert(node);
+    testFunc(node);
+    MongoRunner.stopMongod(node);
+};
+
+/**
+ * Assert that starting MongoDB with --replSet succeeds. Uses a provided testFunc to run any
+ * caller-provided checks on the started node.
+ *
+ * Returns the started node.
+ */
+let assertStartInReplSet = function(replSet, originalNode, cleanData, expectResync, testFunc) {
+    jsTestLog("The node should rejoin the replica set. Clean data: " + cleanData,
+              ". Expect resync: " + expectResync);
+    let node = replSet.start(
+        originalNode, {dbpath: originalNode.dbpath, port: originalNode.port, restart: !cleanData});
+
+    replSet.waitForState(node, ReplSetTest.State.SECONDARY);
+
+    // Ensure that an initial sync attempt was made and succeeded if the data directory was cleaned.
+    let res = assert.commandWorked(node.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    if (expectResync) {
+        assert.eq(1, res.initialSyncStatus.initialSyncAttempts.length);
+        assert.eq(0, res.initialSyncStatus.failedInitialSyncAttempts);
+    } else {
+        assert.eq(undefined, res.initialSyncStatus);
+    }
+
+    testFunc(node);
+    return node;
+};
+
+/**
  * Assert certain error messages are thrown on startup when files are missing or corrupt.
  */
 let assertErrorOnStartupWhenFilesAreCorruptOrMissing = function(
@@ -32,7 +121,7 @@ let assertErrorOnStartupWhenFilesAreCorruptOrMissing = function(
     const mongod = MongoRunner.runMongod({dbpath: dbpath, cleanData: true});
     const testColl = mongod.getDB(dbName)[collName];
     const doc = {a: 1};
-    assert.writeOK(testColl.insert(doc));
+    assert.commandWorked(testColl.insert(doc));
 
     // Stop MongoDB and corrupt/delete certain files.
     deleteOrCorruptFunc(mongod, testColl);
@@ -53,7 +142,7 @@ let assertErrorOnRequestWhenFilesAreCorruptOrMissing = function(
     mongod = MongoRunner.runMongod({dbpath: dbpath, cleanData: true});
     testColl = mongod.getDB(dbName)[collName];
     const doc = {a: 1};
-    assert.writeOK(testColl.insert(doc));
+    assert.commandWorked(testColl.insert(doc));
 
     // Stop MongoDB and corrupt/delete certain files.
     deleteOrCorruptFunc(mongod, testColl);
diff --git a/jstests/disk/repair_does_not_invalidate_config_on_standalone.js b/jstests/disk/repair_does_not_invalidate_config_on_standalone.js
new file mode 100644
index 00000000000..70f0cde97b8
--- /dev/null
+++ b/jstests/disk/repair_does_not_invalidate_config_on_standalone.js
@@ -0,0 +1,42 @@
+/**
+ * Tests that corruption on a standalone does not create a replica set configuration document.
+ *
+ * @tags: [requires_wiredtiger]
+ */
+
+(function() {
+
+    load('jstests/disk/libs/wt_file_helper.js');
+
+    const baseName = "repair_does_not_invalidate_config_on_standalone";
+    const dbName = baseName;
+    const collName = "test";
+
+    const dbpath = MongoRunner.dataPath + baseName + "/";
+    resetDbpath(dbpath);
+
+    let mongod = MongoRunner.runMongod({dbpath: dbpath});
+    const port = mongod.port;
+
+    let testColl = mongod.getDB(dbName)[collName];
+
+    assert.commandWorked(testColl.insert({_id: 0, foo: "bar"}));
+
+    let collUri = getUriForColl(testColl);
+    let collFile = dbpath + "/" + collUri + ".wt";
+
+    MongoRunner.stopMongod(mongod);
+
+    jsTestLog("Deleting collection file: " + collFile);
+    removeFile(collFile);
+
+    assertRepairSucceeds(dbpath, port);
+
+    assertStartAndStopStandaloneOnExistingDbpath(dbpath, port, function(node) {
+        let nodeDB = node.getDB(dbName);
+        assert(nodeDB[collName].exists());
+        assert.eq(nodeDB[collName].find().itcount(), 0);
+
+        assert(!nodeDB.getSiblingDB("local")["system.replset"].exists());
+    });
+})();
diff --git a/jstests/disk/repair_failure_is_recoverable.js b/jstests/disk/repair_failure_is_recoverable.js
new file mode 100644
index 00000000000..cbfc12e5b4b
--- /dev/null
+++ b/jstests/disk/repair_failure_is_recoverable.js
@@ -0,0 +1,62 @@
+/**
+ * This test ensures that a failure during repair does not allow MongoDB to start normally and
+ * requires it to be restarted with --repair again.
+ *
+ * This is not storage-engine specific.
+ */
+
+(function() {
+
+    load('jstests/disk/libs/wt_file_helper.js');
+
+    const exitBeforeRepairParameter = "exitBeforeDataRepair";
+    const exitBeforeRepairInvalidatesConfigParameter = "exitBeforeRepairInvalidatesConfig";
+
+    const baseName = "repair_failure_is_recoverable";
+    const dbName = "repair_failure_is_recoverable";
+    const collName = "test";
+
+    const dbpath = MongoRunner.dataPath + baseName + "/";
+    resetDbpath(dbpath);
+
+    let mongod = MongoRunner.runMongod({dbpath: dbpath});
+    const port = mongod.port;
+
+    let testColl = mongod.getDB(dbName)[collName];
+
+    assert.commandWorked(testColl.insert({_id: 0, foo: "bar"}));
+
+    MongoRunner.stopMongod(mongod);
+
+    /**
+     * Test 1. Cause an exit before repairing data. MongoDB should not be able to restart without
+     * --repair.
+     */
+    assertRepairFailsWithFailpoint(dbpath, port, exitBeforeRepairParameter);
+
+    assertErrorOnStartupAfterIncompleteRepair(dbpath, port);
+
+    assertRepairSucceeds(dbpath, port);
+
+    assertStartAndStopStandaloneOnExistingDbpath(dbpath, port, function(node) {
+        let nodeDB = node.getDB(dbName);
+        assert(nodeDB[collName].exists());
+        assert.eq(nodeDB[collName].find().itcount(), 1);
+    });
+
+    /**
+     * Test 2. Fail after repairing data, before invalidating the replica set config. MongoDB should
+     * not be able to restart without --repair.
+     */
+    assertRepairFailsWithFailpoint(dbpath, port, exitBeforeRepairInvalidatesConfigParameter);
+
+    assertErrorOnStartupAfterIncompleteRepair(dbpath, port);
+
+    assertRepairSucceeds(dbpath, port);
+
+    assertStartAndStopStandaloneOnExistingDbpath(dbpath, port, function(node) {
+        let nodeDB = node.getDB(dbName);
+        assert(nodeDB[collName].exists());
+        assert.eq(nodeDB[collName].find().itcount(), 1);
+    });
+})();
diff --git a/jstests/disk/repair_invalidates_replica_set_config.js b/jstests/disk/repair_invalidates_replica_set_config.js
new file mode 100644
index 00000000000..0f953e55718
--- /dev/null
+++ b/jstests/disk/repair_invalidates_replica_set_config.js
@@ -0,0 +1,128 @@
+/**
+ * This test corrupts WiredTiger data files on a secondary, repairs the data, and asserts that the
+ * node is unable to re-join its original replica set without an initial sync.
+ *
+ * @tags: [requires_wiredtiger]
+ */
+
+(function() {
+
+    load('jstests/disk/libs/wt_file_helper.js');
+
+    const dbName = "repair_invalidates_replica_set_config";
+    const collName = "test";
+
+    let replSet = new ReplSetTest({nodes: 2});
+    replSet.startSet();
+    replSet.initiate();
+    replSet.awaitReplication();
+
+    const originalSecondary = replSet.getSecondary();
+
+    let primaryDB = replSet.getPrimary().getDB(dbName);
+    let secondaryDB = originalSecondary.getDB(dbName);
+
+    assert.commandWorked(primaryDB[collName].insert({_id: 0, foo: "bar"}));
+    replSet.awaitLastOpCommitted();
+
+    const secondaryPort = originalSecondary.port;
+    const secondaryDbpath = originalSecondary.dbpath;
+
+    let secondary = originalSecondary;
+
+    //
+    // 1. This repairs the data on a clean data directory and asserts that the node is still able
+    // to re-join its original replica set without an initial sync.
+    //
+
+    // Shut down the secondary.
+    MongoRunner.stopMongod(secondary);
+
+    // Ensure the secondary can be repaired successfully.
+    assertRepairSucceeds(secondaryDbpath, secondaryPort);
+
+    // Starting up without --replSet should not fail, and the collection should exist with its data.
+    assertStartAndStopStandaloneOnExistingDbpath(secondaryDbpath, secondaryPort, function(node) {
+        let nodeDB = node.getDB(dbName);
+        assert(nodeDB[collName].exists());
+        assert.eq(nodeDB[collName].find().itcount(), 1);
+    });
+
+    // Starting the secondary with the same data directory should succeed with the same data.
+    secondary = assertStartInReplSet(replSet,
+                                     originalSecondary,
+                                     false /* cleanData */,
+                                     false /* expectResync */,
+                                     function(node) {
+                                         let nodeDB = node.getDB(dbName);
+                                         assert.eq(nodeDB[collName].find().itcount(), 1);
+                                     });
+    secondaryDB = secondary.getDB(dbName);
+
+    //
+    // 2. This test corrupts WiredTiger data files on a secondary, repairs the data, and asserts
+    // that the node is unable to re-join its original replica set without an initial sync.
+    //
+
+    let secondaryCollUri = getUriForColl(secondaryDB[collName]);
+    let secondaryCollFile = secondaryDbpath + "/" + secondaryCollUri + ".wt";
+    // Shut down the secondary. Delete the collection's data file.
+    MongoRunner.stopMongod(secondary);
+    jsTestLog("Deleting secondary collection file: " + secondaryCollFile);
+    removeFile(secondaryCollFile);
+
+    // Ensure the secondary can be repaired successfully.
+    assertRepairSucceeds(secondaryDbpath, secondaryPort);
+
+    // Starting up with --replSet should fail with a specific error.
+    assertErrorOnStartupWhenStartingAsReplSet(
+        secondaryDbpath, secondaryPort, replSet.getReplSetConfig()._id);
+
+    // Starting up without --replSet should not fail, but the collection should exist with no data.
+    assertStartAndStopStandaloneOnExistingDbpath(secondaryDbpath, secondaryPort, function(node) {
+        let nodeDB = node.getDB(dbName);
+        assert(nodeDB[collName].exists());
+        assert.eq(nodeDB[collName].find().itcount(), 0);
+    });
+
+    // Starting the secondary with a wiped data directory should force an initial sync.
+    secondary = assertStartInReplSet(
+        replSet, originalSecondary, true /* cleanData */, true /* expectResync */, function(node) {
+            let nodeDB = node.getDB(dbName);
+            assert.eq(nodeDB[collName].find().itcount(), 1);
+        });
+    secondaryDB = secondary.getDB(dbName);
+
+    //
+    // 3. This test corrupts the _mdb_catalog file on a secondary, repairs the data. Because the
+    // node's local.system.replset collection gets deleted, we assert that it is able to start up
+    // and re-sync on the existing data directory immediately.
+
+    // Shut down the secondary. Delete the catalog file.
+    MongoRunner.stopMongod(secondary);
+    let mdbCatalogFile = secondaryDbpath + "/_mdb_catalog.wt";
+    jsTestLog("Deleting secondary catalog file: " + mdbCatalogFile);
+    removeFile(mdbCatalogFile);
+
+    // Ensure the secondary can be repaired successfully.
+    assertRepairSucceeds(secondaryDbpath, secondaryPort);
+
+    // Starting up without --replSet should not fail, but the collection should exist with no data.
+    assertStartAndStopStandaloneOnExistingDbpath(secondaryDbpath, secondaryPort, function(node) {
+        let nodeDB = node.getDB(dbName);
+        assert(!nodeDB[collName].exists());
+        assert(!nodeDB.getSiblingDB("local")["system.replset"].exists());
+    });
+
+    // The node's local.system.replset collection has been deleted, so it's perfectly okay that it
+    // is is able to start up and re-sync.
+    // Starting the secondary with the same data directory should force an initial sync.
+    secondary = assertStartInReplSet(
+        replSet, originalSecondary, false /* cleanData */, true /* expectResync */, function(node) {
+            let nodeDB = node.getDB(dbName);
+            assert.eq(nodeDB[collName].find().itcount(), 1);
+        });
+
+    replSet.stopSet();
+
+})();