SERVER-35782 Repair should move aside unsalvagable data files and create empty ones in their place.

3 files changed, 160 insertions, 27 deletions

diff --git a/jstests/disk/libs/wt_file_helper.js b/jstests/disk/libs/wt_file_helper.js
index 259b9a04bcd..713f5c2e40b 100644
--- a/jstests/disk/libs/wt_file_helper.js
+++ b/jstests/disk/libs/wt_file_helper.js
@@ -24,11 +24,42 @@ let corruptFile = function(file) {
 };
 
 /**
+ * Starts a mongod on the provided data path without clearing data. Accepts 'options' as parameters
+ * to runMongod.
+ */
+let startMongodOnExistingPath = function(dbpath, options) {
+    let args = {dbpath: dbpath, noCleanData: true};
+    for (let attr in options) {
+        if (options.hasOwnProperty(attr))
+            args[attr] = options[attr];
+    }
+    return MongoRunner.runMongod(args);
+};
+
+let assertQueryUsesIndex = function(coll, query, indexName) {
+    let res = coll.find(query).explain();
+    assert.commandWorked(res);
+
+    let inputStage = res.queryPlanner.winningPlan.inputStage;
+    assert.eq(inputStage.stage, "IXSCAN");
+    assert.eq(inputStage.indexName, indexName);
+};
+
+/**
  * Assert that running MongoDB with --repair on the provided dbpath exits cleanly.
  */
-let assertRepairSucceeds = function(dbpath, port) {
+let assertRepairSucceeds = function(dbpath, port, opts) {
+    let args = ["mongod", "--repair", "--port", port, "--dbpath", dbpath];
+    for (let a in opts) {
+        if (opts.hasOwnProperty(a))
+            args.push("--" + a);
+
+        if (opts[a].length > 0) {
+            args.push(a);
+        }
+    }
     jsTestLog("Repairing the node");
-    assert.eq(0, runMongoProgram("mongod", "--repair", "--port", port, "--dbpath", dbpath));
+    assert.eq(0, runMongoProgram.apply(this, args));
 };
 
 let assertRepairFailsWithFailpoint = function(dbpath, port, failpoint) {
diff --git a/jstests/disk/wt_repair_corrupt_files.js b/jstests/disk/wt_repair_corrupt_files.js
new file mode 100644
index 00000000000..331828ed763
--- /dev/null
+++ b/jstests/disk/wt_repair_corrupt_files.js
@@ -0,0 +1,114 @@
+/**
+ * Tests that --repair on WiredTiger correctly and gracefully handles corrupt data files and
+ * directories.
+ *
+ * @tags: [requires_wiredtiger]
+ */
+
+(function() {
+
+    load('jstests/disk/libs/wt_file_helper.js');
+
+    const baseName = "wt_repair_corrupt_files";
+    const collName = "test";
+    const dbpath = MongoRunner.dataPath + baseName + "/";
+
+    /**
+     * Run the test by supplying additional paramters to MongoRunner.runMongod with 'mongodOptions'.
+     */
+    let runTest = function(mongodOptions) {
+        resetDbpath(dbpath);
+        jsTestLog("Running test with args: " + tojson(mongodOptions));
+
+        /**
+         * Test 1. Create a collection, corrupt its .wt file in an unrecoverable way, run repair.
+         * Verify that repair succeeds at rebuilding it. An empty collection should be visible on
+         * normal startup.
+         */
+
+        let mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+        let testColl = mongod.getDB(baseName)[collName];
+
+        const doc = {a: 1};
+        assert.commandWorked(testColl.insert(doc));
+
+        let testCollUri = getUriForColl(testColl);
+        let testCollFile = dbpath + testCollUri + ".wt";
+
+        MongoRunner.stopMongod(mongod);
+
+        jsTestLog("corrupting collection file: " + testCollFile);
+        corruptFile(testCollFile);
+
+        assertRepairSucceeds(dbpath, mongod.port, mongodOptions);
+
+        mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+        testColl = mongod.getDB(baseName)[collName];
+
+        assert.eq(testCollUri, getUriForColl(testColl));
+        assert.eq(testColl.find({}).itcount(), 0);
+        assert.eq(testColl.count(), 0);
+
+        /**
+         * Test 2. Corrupt an index file in an unrecoverable way. Verify that repair rebuilds and
+         * allows MongoDB to start up normally.
+         */
+
+        assert.commandWorked(testColl.insert(doc));
+
+        const indexName = "a_1";
+        assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName}));
+        assertQueryUsesIndex(testColl, doc, indexName);
+
+        let indexUri = getUriForIndex(testColl, indexName);
+
+        MongoRunner.stopMongod(mongod);
+
+        let indexFile = dbpath + indexUri + ".wt";
+        jsTestLog("corrupting index file: " + indexFile);
+        corruptFile(indexFile);
+
+        assertRepairSucceeds(dbpath, mongod.port, mongodOptions);
+        mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+        testColl = mongod.getDB(baseName)[collName];
+
+        // Repair creates new idents.
+        assert.neq(indexUri, getUriForIndex(testColl, indexName));
+
+        assertQueryUsesIndex(testColl, doc, indexName);
+        assert.eq(testColl.find(doc).itcount(), 1);
+        assert.eq(testColl.count(), 1);
+
+        MongoRunner.stopMongod(mongod);
+
+        /**
+         * Test 3. Corrupt the _mdb_catalog in an unrecoverable way. Verify that repair suceeds
+         * in creating an empty catalog and recovers the orphaned testColl, which will still be
+         * accessible in the 'local.system.orphan-' namespace.
+         */
+
+        let mdbCatalogFile = dbpath + "_mdb_catalog.wt";
+        jsTestLog("corrupting catalog file: " + mdbCatalogFile);
+        corruptFile(mdbCatalogFile);
+
+        assertRepairSucceeds(dbpath, mongod.port, mongodOptions);
+
+        mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+        testColl = mongod.getDB(baseName)[collName];
+        assert.isnull(testColl.exists());
+        assert.eq(testColl.find(doc).itcount(), 0);
+        assert.eq(testColl.count(), 0);
+
+        // Ensure the collection orphan was created with the existing document.
+        let orphanColl = mongod.getDB('local').getCollection('system.orphan-' + testCollUri);
+        assert(orphanColl.exists());
+        assert.eq(orphanColl.find(doc).itcount(), 1);
+        assert.eq(orphanColl.count(), 1);
+
+        MongoRunner.stopMongod(mongod);
+    };
+
+    runTest({});
+    runTest({directoryperdb: ""});
+    runTest({wiredTigerDirectoryForIndexes: ""});
+})();
diff --git a/jstests/disk/wt_repair_missing_files.js b/jstests/disk/wt_repair_missing_files.js
index 551d439312a..8529d1f01b3 100644
--- a/jstests/disk/wt_repair_missing_files.js
+++ b/jstests/disk/wt_repair_missing_files.js
@@ -20,11 +20,11 @@
      * re-creating it. The collection should be visible on normal startup.
      */
 
-    let mongod = MongoRunner.runMongod({dbpath: dbpath});
+    let mongod = startMongodOnExistingPath(dbpath);
     let testColl = mongod.getDB(baseName)[collName];
 
     const doc = {a: 1};
-    assert.writeOK(testColl.insert(doc));
+    assert.commandWorked(testColl.insert(doc));
 
     let testCollUri = getUriForColl(testColl);
     let testCollFile = dbpath + testCollUri + ".wt";
@@ -34,9 +34,9 @@
     jsTestLog("deleting collection file: " + testCollFile);
     removeFile(testCollFile);
 
-    assert.eq(0, runMongoProgram("mongod", "--repair", "--port", mongod.port, "--dbpath", dbpath));
+    assertRepairSucceeds(dbpath, mongod.port);
 
-    mongod = MongoRunner.runMongod({dbpath: dbpath, noCleanData: true});
+    mongod = startMongodOnExistingPath(dbpath);
     testColl = mongod.getDB(baseName)[collName];
 
     assert.eq(testCollUri, getUriForColl(testColl));
@@ -48,16 +48,7 @@
      * normally.
      */
 
-    let assertQueryUsesIndex = function(coll, query, indexName) {
-        let res = coll.find(query).explain();
-        assert.commandWorked(res);
-
-        let inputStage = res.queryPlanner.winningPlan.inputStage;
-        assert.eq(inputStage.stage, "IXSCAN");
-        assert.eq(inputStage.indexName, indexName);
-    };
-
-    assert.writeOK(testColl.insert(doc));
+    assert.commandWorked(testColl.insert(doc));
 
     const indexName = "a_1";
     assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName}));
@@ -71,8 +62,8 @@
     jsTestLog("deleting index file: " + indexFile);
     removeFile(indexFile);
 
-    assert.eq(0, runMongoProgram("mongod", "--repair", "--port", mongod.port, "--dbpath", dbpath));
-    mongod = MongoRunner.runMongod({dbpath: dbpath, noCleanData: true});
+    assertRepairSucceeds(dbpath, mongod.port);
+    mongod = startMongodOnExistingPath(dbpath);
     testColl = mongod.getDB(baseName)[collName];
 
     // Repair creates new idents.
@@ -93,9 +84,9 @@
     jsTestLog("deleting catalog file: " + mdbCatalogFile);
     removeFile(mdbCatalogFile);
 
-    assert.eq(0, runMongoProgram("mongod", "--repair", "--port", mongod.port, "--dbpath", dbpath));
+    assertRepairSucceeds(dbpath, mongod.port);
 
-    mongod = MongoRunner.runMongod({dbpath: dbpath, noCleanData: true});
+    mongod = startMongodOnExistingPath(dbpath);
     testColl = mongod.getDB(baseName)[collName];
     assert.isnull(testColl.exists());
 
@@ -110,10 +101,10 @@
     MongoRunner.stopMongod(mongod);
     resetDbpath(dbpath);
 
-    mongod = MongoRunner.runMongod({dbpath: dbpath, directoryperdb: "", noCleanData: true});
+    mongod = startMongodOnExistingPath(dbpath, {directoryperdb: ""});
     testColl = mongod.getDB(baseName)[collName];
 
-    assert.writeOK(testColl.insert(doc));
+    assert.commandWorked(testColl.insert(doc));
 
     testCollUri = getUriForColl(testColl);
 
@@ -123,12 +114,9 @@
     jsTestLog("deleting data directory: " + dataDir);
     removeFile(dataDir);
 
-    assert.eq(
-        0,
-        runMongoProgram(
-            "mongod", "--repair", "--directoryperdb", "--port", mongod.port, "--dbpath", dbpath));
+    assertRepairSucceeds(dbpath, mongod.port, {directoryperdb: ""});
 
-    mongod = MongoRunner.runMongod({dbpath: dbpath, directoryperdb: "", noCleanData: true});
+    mongod = startMongodOnExistingPath(dbpath, {directoryperdb: ""});
     testColl = mongod.getDB(baseName)[collName];
 
     assert.eq(testCollUri, getUriForColl(testColl));