SERVER-35630 Throw error message to run repair when detecting missing or corrupt data files

(cherry picked from commit 3deba1b701332ad9afb6e722db928ca98714ea87)

4 files changed, 230 insertions, 17 deletions

diff --git a/jstests/disk/libs/wt_file_helper.js b/jstests/disk/libs/wt_file_helper.js
new file mode 100644
index 00000000000..f8483e0047f
--- /dev/null
+++ b/jstests/disk/libs/wt_file_helper.js
@@ -0,0 +1,72 @@
+/**
+ * Get the URI of the wt collection file given the collection name.
+ */
+let getUriForColl = function(coll) {
+    assert(coll.exists());  // Collection must exist
+    return coll.stats().wiredTiger.uri.split("table:")[1];
+};
+
+/**
+ * Get the URI of the wt index file given the collection name and the index name.
+ */
+let getUriForIndex = function(coll, indexName) {
+    assert(coll.exists());  // Collection must exist
+    const ret = assert.commandWorked(coll.getDB().runCommand({collStats: coll.getName()}));
+    return ret.indexDetails[indexName].uri.split("table:")[1];
+};
+
+/**
+ * 'Corrupt' the file by replacing it with an empty file.
+ */
+let corruptFile = function(file) {
+    removeFile(file);
+    writeFile(file, "");
+};
+
+/**
+ * Assert certain error messages are thrown on startup when files are missing or corrupt.
+ */
+let assertErrorOnStartupWhenFilesAreCorruptOrMissing = function(
+    dbpath, dbName, collName, deleteOrCorruptFunc, errmsg) {
+    // Start a MongoDB instance, create the collection file.
+    const mongod = MongoRunner.runMongod({dbpath: dbpath, cleanData: true});
+    const testColl = mongod.getDB(dbName)[collName];
+    const doc = {a: 1};
+    assert.writeOK(testColl.insert(doc));
+
+    // Stop MongoDB and corrupt/delete certain files.
+    deleteOrCorruptFunc(mongod, testColl);
+
+    // Restart the MongoDB instance and get an expected error message.
+    clearRawMongoProgramOutput();
+    assert.eq(MongoRunner.EXIT_ABRUPT,
+              runMongoProgram("mongod", "--port", mongod.port, "--dbpath", dbpath));
+    assert.gte(rawMongoProgramOutput().indexOf(errmsg), 0);
+};
+
+/**
+ * Assert certain error messages are thrown on a specific request when files are missing or corrupt.
+ */
+let assertErrorOnRequestWhenFilesAreCorruptOrMissing = function(
+    dbpath, dbName, collName, deleteOrCorruptFunc, requestFunc, errmsg) {
+    // Start a MongoDB instance, create the collection file.
+    mongod = MongoRunner.runMongod({dbpath: dbpath, cleanData: true});
+    testColl = mongod.getDB(dbName)[collName];
+    const doc = {a: 1};
+    assert.writeOK(testColl.insert(doc));
+
+    // Stop MongoDB and corrupt/delete certain files.
+    deleteOrCorruptFunc(mongod, testColl);
+
+    // Restart the MongoDB instance.
+    clearRawMongoProgramOutput();
+    mongod = MongoRunner.runMongod({dbpath: dbpath, port: mongod.port, noCleanData: true});
+
+    // This request crashes the server.
+    testColl = mongod.getDB(dbName)[collName];
+    requestFunc(testColl);
+
+    // Get an expected error message.
+    assert.gte(rawMongoProgramOutput().indexOf(errmsg), 0);
+    MongoRunner.stopMongod(mongod, 9, {allowedExitCode: MongoRunner.EXIT_ABRUPT});
+};
diff --git a/jstests/disk/wt_corrupt_file_errors.js b/jstests/disk/wt_corrupt_file_errors.js
new file mode 100644
index 00000000000..f6e7204ccbf
--- /dev/null
+++ b/jstests/disk/wt_corrupt_file_errors.js
@@ -0,0 +1,78 @@
+/**
+ * Tests that MongoDB gives errors when certain data files are corrupted.
+ *
+ * @tags: [requires_wiredtiger]
+ */
+
+(function() {
+
+    load('jstests/disk/libs/wt_file_helper.js');
+
+    const baseName = "wt_corrupt_file_errors";
+    const collName = "test";
+    const dbpath = MongoRunner.dataPath + baseName + "/";
+
+    /**
+     * Test 1. Corrupt a collection's .wt file.
+     */
+
+    assertErrorOnStartupWhenFilesAreCorruptOrMissing(
+        dbpath, baseName, collName, (mongod, testColl) => {
+            const testCollUri = getUriForColl(testColl);
+            const testCollFile = dbpath + testCollUri + ".wt";
+            MongoRunner.stopMongod(mongod);
+            jsTestLog("corrupting collection file: " + testCollFile);
+            corruptFile(testCollFile);
+        }, "Fatal Assertion 50882");
+
+    /**
+     * Test 2. Corrupt the _mdb_catalog.
+     */
+
+    assertErrorOnStartupWhenFilesAreCorruptOrMissing(
+        dbpath, baseName, collName, (mongod, testColl) => {
+            MongoRunner.stopMongod(mongod);
+            const mdbCatalogFile = dbpath + "_mdb_catalog.wt";
+            jsTestLog("corrupting catalog file: " + mdbCatalogFile);
+            corruptFile(mdbCatalogFile);
+        }, "Fatal Assertion 50882");
+
+    /**
+     * Test 3. Corrupt the WiredTiger.wt.
+     */
+
+    assertErrorOnStartupWhenFilesAreCorruptOrMissing(
+        dbpath, baseName, collName, (mongod, testColl) => {
+            MongoRunner.stopMongod(mongod);
+            const WiredTigerWTFile = dbpath + "WiredTiger.wt";
+            jsTestLog("corrupting WiredTiger.wt");
+            corruptFile(WiredTigerWTFile);
+        }, "Fatal Assertion 28595");
+
+    /**
+     * Test 4. Corrupt an index file.
+     */
+
+    assertErrorOnRequestWhenFilesAreCorruptOrMissing(
+        dbpath,
+        baseName,
+        collName,
+        (mongod, testColl) => {
+            const indexName = "a_1";
+            assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName}));
+            const indexUri = getUriForIndex(testColl, indexName);
+            MongoRunner.stopMongod(mongod);
+            const indexFile = dbpath + indexUri + ".wt";
+            jsTestLog("corrupting index file: " + indexFile);
+            corruptFile(indexFile);
+        },
+        (testColl) => {
+            // This insert will crash the server because it triggers the code path
+            // of looking for the index file.
+            assert.throws(function() {
+                testColl.insert({a: 1});
+            });
+        },
+        "Fatal Assertion 50882");
+
+})();
diff --git a/jstests/disk/wt_missing_file_errors.js b/jstests/disk/wt_missing_file_errors.js
new file mode 100644
index 00000000000..626974bbc30
--- /dev/null
+++ b/jstests/disk/wt_missing_file_errors.js
@@ -0,0 +1,78 @@
+/**
+ * Tests that MongoDB gives errors when certain data files are missing.
+ *
+ * @tags: [requires_wiredtiger]
+ */
+
+(function() {
+
+    load('jstests/disk/libs/wt_file_helper.js');
+
+    const baseName = "wt_missing_file_errors";
+    const collName = "test";
+    const dbpath = MongoRunner.dataPath + baseName + "/";
+
+    /**
+     * Test 1. Delete a collection's .wt file.
+     */
+
+    assertErrorOnStartupWhenFilesAreCorruptOrMissing(
+        dbpath, baseName, collName, (mongod, testColl) => {
+            const testCollUri = getUriForColl(testColl);
+            const testCollFile = dbpath + testCollUri + ".wt";
+            MongoRunner.stopMongod(mongod);
+            jsTestLog("deleting collection file: " + testCollFile);
+            removeFile(testCollFile);
+        }, "Fatal Assertion 50883");
+
+    /**
+     * Test 2. Delete the _mdb_catalog.
+     */
+
+    assertErrorOnStartupWhenFilesAreCorruptOrMissing(
+        dbpath, baseName, collName, (mongod, testColl) => {
+            MongoRunner.stopMongod(mongod);
+            let mdbCatalogFile = dbpath + "_mdb_catalog.wt";
+            jsTestLog("deleting catalog file: " + mdbCatalogFile);
+            removeFile(mdbCatalogFile);
+        }, "Fatal Assertion 50883");
+
+    /**
+     * Test 3. Delete the WiredTiger.wt.
+     */
+
+    assertErrorOnStartupWhenFilesAreCorruptOrMissing(
+        dbpath, baseName, collName, (mongod, testColl) => {
+            MongoRunner.stopMongod(mongod);
+            let WiredTigerWTFile = dbpath + "WiredTiger.wt";
+            jsTestLog("deleting WiredTiger.wt");
+            removeFile(WiredTigerWTFile);
+        }, "Fatal Assertion 28595");
+
+    /**
+     * Test 4. Delete an index file.
+     */
+
+    assertErrorOnRequestWhenFilesAreCorruptOrMissing(
+        dbpath,
+        baseName,
+        collName,
+        (mongod, testColl) => {
+            const indexName = "a_1";
+            assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName}));
+            const indexUri = getUriForIndex(testColl, indexName);
+            MongoRunner.stopMongod(mongod);
+            const indexFile = dbpath + indexUri + ".wt";
+            jsTestLog("deleting index file: " + indexFile);
+            removeFile(indexFile);
+        },
+        (testColl) => {
+            // This insert will crash the server because it triggers the code path
+            // of looking for the index file.
+            assert.throws(function() {
+                testColl.insert({a: 1});
+            });
+        },
+        "Fatal Assertion 50883");
+
+})();
diff --git a/jstests/disk/wt_repair_missing_files.js b/jstests/disk/wt_repair_missing_files.js
index c0273f7b647..fffcab20faf 100644
--- a/jstests/disk/wt_repair_missing_files.js
+++ b/jstests/disk/wt_repair_missing_files.js
@@ -7,29 +7,14 @@
 
 (function() {
 
+    load('jstests/disk/libs/wt_file_helper.js');
+
     const baseName = "wt_repair_missing_files";
     const collName = "test";
     const dbpath = MongoRunner.dataPath + baseName + "/";
 
     resetDbpath(dbpath);
 
-    let getUriForColl = function(coll) {
-        let ret = coll.stats();
-        assert.commandWorked(ret);
-        let uri = ret.wiredTiger.uri.split("table:")[1];
-        assert.neq(typeof(uri), 'undefined');
-        return uri;
-    };
-
-    let getUriForIndex = function(coll, indexName) {
-        let ret = coll.getDB().runCommand({collStats: coll.getName()});
-        assert.commandWorked(ret);
-
-        let uri = ret.indexDetails[indexName].uri.split("table:")[1];
-        assert.neq(typeof(uri), 'undefined');
-        return uri;
-    };
-
     /**
      * Test 1. Create a collection, delete it's .wt file, run repair. Verify that repair succeeds at
      * re-creating it. The collection should be visible on normal startup.