SERVER-16173 Rewrite repair for KVEngines

22 files changed, 388 insertions, 58 deletions

diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index a1979673f24..e50cf8b9f16 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -660,6 +660,7 @@ serverOnlyFiles = [ "db/background.cpp",
                     "db/prefetch.cpp",
                     "db/range_deleter_db_env.cpp",
                     "db/range_deleter_service.cpp",
+                    "db/repair_database.cpp",
                     "db/repl/bgsync.cpp",
                     "db/repl/initial_sync.cpp",
                     "db/repl/master_slave.cpp",
diff --git a/src/mongo/client/redef_macros.h b/src/mongo/client/redef_macros.h
index 8a4f5ffe4cb..5ce89464498 100644
--- a/src/mongo/client/redef_macros.h
+++ b/src/mongo/client/redef_macros.h
@@ -37,7 +37,7 @@
 // util/assert_util.h
 #pragma push_macro("verify")
 #undef verify
-#define verify MONGO_verify
+#define verify(expression) MONGO_verify(expression)
 #pragma push_macro("invariant")
 #undef invariant
 #define invariant MONGO_invariant
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 76fe8f18e47..b7693bdfe16 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -71,6 +71,7 @@
 #include "mongo/db/operation_context_impl.h"
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/range_deleter_service.h"
+#include "mongo/db/repair_database.h"
 #include "mongo/db/repl/network_interface_impl.h"
 #include "mongo/db/repl/repl_coordinator_external_state_impl.h"
 #include "mongo/db/repl/repl_coordinator_global.h"
@@ -338,7 +339,7 @@ namespace mongo {
                 const string dbName = *i;
                 LOG(1) << "    Repairing database: " << dbName << endl;
 
-                fassert(18506, storageEngine->repairDatabase(&txn, dbName));
+                fassert(18506, repairDatabase(&txn, storageEngine, dbName));
             }
         }
 
diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp
index 08b3b687c56..bb8533d0f68 100644
--- a/src/mongo/db/dbcommands.cpp
+++ b/src/mongo/db/dbcommands.cpp
@@ -68,6 +68,7 @@
 #include "mongo/db/query/get_executor.h"
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/query/query_planner.h"
+#include "mongo/db/repair_database.h"
 #include "mongo/db/repl/repl_coordinator_global.h"
 #include "mongo/db/repl/repl_settings.h"
 #include "mongo/db/repl/oplog.h"
@@ -285,8 +286,9 @@ namespace mongo {
             e = cmdObj.getField( "backupOriginalFiles" );
             bool backupOriginalFiles = e.isBoolean() && e.boolean();
 
-            Status status = getGlobalEnvironment()->getGlobalStorageEngine()->repairDatabase(
-                txn, dbname, preserveClonedFilesOnFailure, backupOriginalFiles );
+            StorageEngine* engine = getGlobalEnvironment()->getGlobalStorageEngine();
+            Status status = repairDatabase(txn, engine, dbname, preserveClonedFilesOnFailure,
+                                           backupOriginalFiles );
 
             IndexBuilder::restoreIndexes(indexesInProg);
 
diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp
new file mode 100644
index 00000000000..7a133591a01
--- /dev/null
+++ b/src/mongo/db/repair_database.cpp
@@ -0,0 +1,223 @@
+/**
+*    Copyright (C) 2014 MongoDB Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+*    As a special exception, the copyright holders give permission to link the
+*    code of portions of this program with the OpenSSL library under certain
+*    conditions as described in each individual source file and distribute
+*    linked combinations including the program with the OpenSSL library. You
+*    must comply with the GNU Affero General Public License in all respects for
+*    all of the code used other than as permitted herein. If you modify file(s)
+*    with this exception, you may extend this exception to your version of the
+*    file(s), but you are not obligated to do so. If you do not wish to do so,
+*    delete this exception statement from your version. If you delete this
+*    exception statement from all source files in the program, then also delete
+*    it in the license file.
+*/
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/repair_database.h"
+
+#include <boost/scoped_ptr.hpp>
+
+#include "mongo/db/background.h"
+#include "mongo/base/status.h"
+#include "mongo/base/string_data.h"
+#include "mongo/bson/bson_validate.h"
+#include "mongo/db/catalog/collection.h"
+#include "mongo/db/catalog/collection_catalog_entry.h"
+#include "mongo/db/catalog/database.h"
+#include "mongo/db/catalog/database_catalog_entry.h"
+#include "mongo/db/catalog/database_holder.h"
+#include "mongo/db/catalog/index_create.h"
+#include "mongo/db/catalog/index_key_validate.h"
+#include "mongo/db/storage/mmap_v1/mmap_v1_engine.h"
+#include "mongo/db/storage/storage_engine.h"
+#include "mongo/util/log.h"
+
+namespace mongo {
+namespace {
+    Status rebuildIndexesOnCollection(OperationContext* txn,
+                                      DatabaseCatalogEntry* dbce,
+                                      const std::string& collectionName) {
+
+        Database db(dbce->name(), dbce);
+        CollectionCatalogEntry* cce = dbce->getCollectionCatalogEntry(txn, collectionName);
+
+        std::vector<string> indexNames;
+        std::vector<BSONObj> indexSpecs;
+        {
+            // Fetch all indexes
+            cce->getAllIndexes( txn, &indexNames );
+            for ( size_t i = 0; i < indexNames.size(); i++ ) {
+                const string& name = indexNames[i];
+                BSONObj spec = cce->getIndexSpec( txn, name );
+                indexSpecs.push_back(spec.removeField("v").getOwned());
+
+                const BSONObj key = spec.getObjectField("key");
+                const Status keyStatus = validateKeyPattern(key);
+                if (!keyStatus.isOK()) {
+                    return Status(ErrorCodes::CannotCreateIndex, str::stream()
+                        << "Cannot rebuild index " << spec << ": " << keyStatus.reason()
+                        << " For more info see http://dochub.mongodb.org/core/index-validation");
+                }
+            }
+        }
+
+        // Skip the rest if there are no indexes to rebuild.
+        if (indexSpecs.empty()) return Status::OK();
+
+        Collection* collection;
+        boost::scoped_ptr<MultiIndexBlock> indexer;
+        {
+            // These steps are combined into a single WUOW to ensure there are no commits without
+            // the indexes.
+            // 1) Drop all indexes.
+            // 2) Open the Collection
+            // 3) Start the index build process.
+
+            WriteUnitOfWork wuow(txn);
+
+            { // 1
+                for ( size_t i = 0; i < indexNames.size(); i++ ) {
+                    Status s = cce->removeIndex(txn, indexNames[i]);
+                    if (!s.isOK()) return s;
+                }
+            }
+
+            // Indexes must be dropped before we open the Collection otherwise we could attempt to
+            // open a bad index and fail.
+            // TODO see if MultiIndexBlock can be made to work without a Collection.
+            collection = db.getCollection(txn, collectionName);
+            invariant(collection);
+
+            indexer.reset(new MultiIndexBlock(txn, collection));
+            Status status = indexer->init(indexSpecs);
+            if (!status.isOK()) {
+                // The WUOW will handle cleanup, so the indexer shouldn't do its own.
+                indexer->abortWithoutCleanup();
+                return status;
+            }
+
+            wuow.commit();
+        }
+
+        // Iterate all records in the collection. Delete them if they aren't valid BSON. Index them
+        // if they are.
+
+        RecordStore* rs = collection->getRecordStore();
+        boost::scoped_ptr<RecordIterator> it(rs->getIterator(txn));
+        while (!it->isEOF()) {
+            RecordId id = it->curr();
+            RecordData data = it->dataFor(id);
+            invariant(id == it->getNext());
+
+            Status status = validateBSON(data.data(), data.size());
+            if (!status.isOK()) {
+                log() << "Invalid BSON detected at " << id << ": " << status << ". Deleting.";
+                it->saveState();
+                {
+                    WriteUnitOfWork wunit(txn);
+                    rs->deleteRecord(txn, id);
+                    wunit.commit();
+                }
+                it->restoreState(txn);
+                continue;
+            }
+
+            // Now index the record.
+            // TODO SERVER-14812 add a mode that drops duplicates rather than failing
+            WriteUnitOfWork wunit(txn);
+            status = indexer->insert(data.releaseToBson(), id);
+            if (!status.isOK()) return status;
+            wunit.commit();
+        }
+
+        Status status = indexer->doneInserting();
+        if (!status.isOK()) return status;
+
+        {
+            WriteUnitOfWork wunit(txn);
+            indexer->commit();
+            wunit.commit();
+        }
+
+        return Status::OK();
+    }
+} // namespace
+
+    Status repairDatabase(OperationContext* txn,
+                          StorageEngine* engine,
+                          const std::string& dbName,
+                          bool preserveClonedFilesOnFailure,
+                          bool backupOriginalFiles) {
+
+        // We must hold some form of lock here
+        invariant(txn->lockState()->isLocked());
+        invariant( dbName.find( '.' ) == string::npos );
+
+        log() << "repairDatabase " << dbName << endl;
+
+        BackgroundOperation::assertNoBgOpInProgForDb(dbName);
+
+        txn->checkForInterrupt();
+
+        if (engine->isMmapV1()) {
+            // MMAPv1 is a layering violation so it implements its own repairDatabase.
+            return static_cast<MMAPV1Engine*>(engine)->repairDatabase(txn,
+                                                                      dbName,
+                                                                      preserveClonedFilesOnFailure,
+                                                                      backupOriginalFiles);
+        }
+
+        // These are MMAPv1 specific
+        if ( preserveClonedFilesOnFailure ) {
+            return Status( ErrorCodes::BadValue, "preserveClonedFilesOnFailure not supported" );
+        }
+        if ( backupOriginalFiles ) {
+            return Status( ErrorCodes::BadValue, "backupOriginalFiles not supported" );
+        }
+
+        // Close and reopen the db to invalidate all current users and caches.
+        // WARNING: it is important that the opened Database object isn't used for anything as its
+        // cache must remain empty until we return.
+        dbHolder().close(txn, dbName);
+        dbHolder().openDb(txn, dbName);
+
+        DatabaseCatalogEntry* dbce = engine->getDatabaseCatalogEntry(txn, dbName);
+
+        std::list<std::string> colls;
+        dbce->getCollectionNamespaces(&colls);
+
+        for (std::list<std::string>::const_iterator it = colls.begin(); it != colls.end(); ++it) {
+            // Don't check for interrupt after starting to repair a collection otherwise we can
+            // leave data in an inconsistent state. Interrupting between collections is ok, however.
+            txn->checkForInterrupt();
+
+            log() << "Repairing collection " << *it;
+
+            Status status = engine->repairRecordStore(txn, *it);
+            if (!status.isOK()) return status;
+
+            status = rebuildIndexesOnCollection(txn, dbce, *it);
+            if (!status.isOK()) return status;
+        }
+
+        return Status::OK();
+    }
+}
+
diff --git a/src/mongo/db/repair_database.h b/src/mongo/db/repair_database.h
new file mode 100644
index 00000000000..85d2729a277
--- /dev/null
+++ b/src/mongo/db/repair_database.h
@@ -0,0 +1,45 @@
+/**
+*    Copyright (C) 2014 MongoDB Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+*    As a special exception, the copyright holders give permission to link the
+*    code of portions of this program with the OpenSSL library under certain
+*    conditions as described in each individual source file and distribute
+*    linked combinations including the program with the OpenSSL library. You
+*    must comply with the GNU Affero General Public License in all respects for
+*    all of the code used other than as permitted herein. If you modify file(s)
+*    with this exception, you may extend this exception to your version of the
+*    file(s), but you are not obligated to do so. If you do not wish to do so,
+*    delete this exception statement from your version. If you delete this
+*    exception statement from all source files in the program, then also delete
+*    it in the license file.
+*/
+
+#pragma once
+
+#include <string>
+
+namespace mongo {
+    class OperationContext;
+    class Status;
+    class StorageEngine;
+    class StringData;
+
+    Status repairDatabase(OperationContext* txn,
+                          StorageEngine* engine,
+                          const std::string& dbName,
+                          bool preserveClonedFilesOnFailure = false,
+                          bool backupOriginalFiles = false);
+}
+
diff --git a/src/mongo/db/storage/devnull/devnull_init.cpp b/src/mongo/db/storage/devnull/devnull_init.cpp
index aa752c3da07..5e36cc51589 100644
--- a/src/mongo/db/storage/devnull/devnull_init.cpp
+++ b/src/mongo/db/storage/devnull/devnull_init.cpp
@@ -42,7 +42,10 @@ namespace mongo {
         class DevNullStorageEngineFactory : public StorageEngine::Factory {
         public:
             virtual StorageEngine* create( const StorageGlobalParams& params ) const {
-                return new KVStorageEngine( new DevNullKVEngine() );
+                KVStorageEngineOptions options;
+                options.directoryPerDB = params.directoryperdb;
+                options.forRepair = params.repair;
+                return new KVStorageEngine( new DevNullKVEngine(), options );
             }
 
             virtual StringData getCanonicalName() const {
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index 473dcebfbab..626ce33d2ea 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -88,6 +88,10 @@ namespace mongo {
             return Status::OK();
         }
 
+        virtual bool hasIdent(OperationContext* opCtx, const StringData& ident) const {
+            return true;
+        }
+
         std::vector<std::string> getAllIdents( OperationContext* opCtx ) const {
             return std::vector<std::string>();
         }
diff --git a/src/mongo/db/storage/in_memory/in_memory_engine.h b/src/mongo/db/storage/in_memory/in_memory_engine.h
index 0c913cf8296..a466bd93dc3 100644
--- a/src/mongo/db/storage/in_memory/in_memory_engine.h
+++ b/src/mongo/db/storage/in_memory/in_memory_engine.h
@@ -80,6 +80,10 @@ namespace mongo {
 
         virtual void cleanShutdown() {};
 
+        virtual bool hasIdent(OperationContext* opCtx, const StringData& ident) const {
+            return _dataMap.find(ident) != _dataMap.end();;
+        }
+
         std::vector<std::string> getAllIdents( OperationContext* opCtx ) const;
     private:
         typedef StringMap<boost::shared_ptr<void> > DataMap;
diff --git a/src/mongo/db/storage/in_memory/in_memory_init.cpp b/src/mongo/db/storage/in_memory/in_memory_init.cpp
index c0dc832459f..a007acaa3da 100644
--- a/src/mongo/db/storage/in_memory/in_memory_init.cpp
+++ b/src/mongo/db/storage/in_memory/in_memory_init.cpp
@@ -43,7 +43,10 @@ namespace mongo {
         public:
             virtual ~InMemoryFactory() { }
             virtual StorageEngine* create(const StorageGlobalParams& params) const {
-                return new KVStorageEngine(new InMemoryEngine());
+                KVStorageEngineOptions options;
+                options.directoryPerDB = params.directoryperdb;
+                options.forRepair = params.repair;
+                return new KVStorageEngine(new InMemoryEngine(), options);
             }
 
             virtual StringData getCanonicalName() const {
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 9f343e1238c..e7eacddf1ca 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -110,6 +110,8 @@ namespace mongo {
             return Status::OK();
         }
 
+        virtual bool hasIdent(OperationContext* opCtx, const StringData& ident) const = 0;
+
         virtual std::vector<std::string> getAllIdents( OperationContext* opCtx ) const = 0;
 
         /**
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp
index a1ec8b1e53c..ff85f6c48d6 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp
@@ -72,6 +72,13 @@ namespace mongo {
         , _supportsDocLocking(_engine->supportsDocLocking()) {
 
         OperationContextNoop opCtx( _engine->newRecoveryUnit() );
+
+        if (options.forRepair && engine->hasIdent(&opCtx, catalogInfo)) {
+            log() << "Repairing catalog metadata";
+            // TODO should also validate all BSON in the catalog.
+            engine->repairIdent(&opCtx, catalogInfo);
+        }
+
         {
             WriteUnitOfWork uow( &opCtx );
 
@@ -251,24 +258,7 @@ namespace mongo {
         return _engine->isDurable();
     }
 
-    Status KVStorageEngine::repairDatabase( OperationContext* txn,
-                                            const std::string& dbName,
-                                            bool preserveClonedFilesOnFailure,
-                                            bool backupOriginalFiles ) {
-        if ( preserveClonedFilesOnFailure ) {
-            return Status( ErrorCodes::BadValue, "preserveClonedFilesOnFailure not supported" );
-        }
-        if ( backupOriginalFiles ) {
-            return Status( ErrorCodes::BadValue, "backupOriginalFiles not supported" );
-        }
-
-        vector<string> idents = _catalog->getAllIdentsForDB( dbName );
-        for ( size_t i = 0; i < idents.size(); i++ ) {
-            Status status = _engine->repairIdent( txn, idents[i] );
-            if ( !status.isOK() )
-                return status;
-        }
-        return Status::OK();
+    Status KVStorageEngine::repairRecordStore(OperationContext* txn, const std::string& ns) {
+        return _engine->repairIdent(txn, _catalog->getCollectionIdent(ns));
     }
-
 }
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h
index 5e52ca1c3a0..562dd2dba61 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.h
+++ b/src/mongo/db/storage/kv/kv_storage_engine.h
@@ -49,10 +49,12 @@ namespace mongo {
     struct KVStorageEngineOptions {
         KVStorageEngineOptions() :
             directoryPerDB(false),
-            directoryForIndexes(false) {}
+            directoryForIndexes(false),
+            forRepair(false) {}
 
         bool directoryPerDB;
         bool directoryForIndexes;
+        bool forRepair;
     };
 
     class KVStorageEngine : public StorageEngine {
@@ -83,10 +85,7 @@ namespace mongo {
 
         virtual bool isDurable() const;
 
-        virtual Status repairDatabase( OperationContext* txn,
-                                       const std::string& dbName,
-                                       bool preserveClonedFilesOnFailure = false,
-                                       bool backupOriginalFiles = false );
+        virtual Status repairRecordStore(OperationContext* txn, const std::string& ns);
 
         virtual void cleanShutdown();
 
diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h
index b7fab00d07b..cfc49e8e7e7 100644
--- a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h
+++ b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.h
@@ -51,11 +51,6 @@ namespace mongo {
         void listDatabases( std::vector<std::string>* out ) const;
         int flushAllFiles( bool sync );
 
-        Status repairDatabase( OperationContext* tnx,
-                               const std::string& dbName,
-                               bool preserveClonedFilesOnFailure,
-                               bool backupOriginalFiles );
-
         DatabaseCatalogEntry* getDatabaseCatalogEntry( OperationContext* opCtx,
                                                        const StringData& db );
 
@@ -70,6 +65,16 @@ namespace mongo {
 
         virtual void cleanShutdown();
 
+        // Callers should use  repairDatabase instead.
+        virtual Status repairRecordStore(OperationContext* txn, const std::string& ns) {
+            return Status(ErrorCodes::InternalError, "MMAPv1 doesn't support repairRecordStore");
+        }
+
+        // MMAPv1 specific (non-virtual)
+        Status repairDatabase( OperationContext* tnx,
+                               const std::string& dbName,
+                               bool preserveClonedFilesOnFailure,
+                               bool backupOriginalFiles );
     private:
         static void _listDatabases( const std::string& directory,
                                     std::vector<std::string>* out );
diff --git a/src/mongo/db/storage/mmap_v1/repair_database.cpp b/src/mongo/db/storage/mmap_v1/repair_database.cpp
index f04ac9379f0..366549f0ff0 100644
--- a/src/mongo/db/storage/mmap_v1/repair_database.cpp
+++ b/src/mongo/db/storage/mmap_v1/repair_database.cpp
@@ -272,16 +272,8 @@ namespace mongo {
                                          const std::string& dbName,
                                          bool preserveClonedFilesOnFailure,
                                          bool backupOriginalFiles ) {
-        // We must hold some form of lock here
-        invariant(txn->lockState()->isLocked());
-        invariant( dbName.find( '.' ) == string::npos );
-
         scoped_ptr<RepairFileDeleter> repairFileDeleter;
 
-        log() << "repairDatabase " << dbName << endl;
-
-        BackgroundOperation::assertNoBgOpInProgForDb(dbName);
-
         // Must be done before and after repair
         getDur().syncDataAndTruncateJournal(txn);
 
diff --git a/src/mongo/db/storage/rocks/rocks_engine.h b/src/mongo/db/storage/rocks/rocks_engine.h
index ca8ef727325..f3215885771 100644
--- a/src/mongo/db/storage/rocks/rocks_engine.h
+++ b/src/mongo/db/storage/rocks/rocks_engine.h
@@ -90,6 +90,9 @@ namespace mongo {
 
         virtual Status dropIdent(OperationContext* opCtx, const StringData& ident) override;
 
+        virtual bool hasIdent(OperationContext* opCtx, const StringData& ident) const {
+            return _identColumnFamilyMap.find(ident) != _identColumnFamilyMap.end();;
+        }
         virtual std::vector<std::string> getAllIdents( OperationContext* opCtx ) const override;
 
         virtual bool supportsDocLocking() const override {
diff --git a/src/mongo/db/storage/rocks/rocks_init.cpp b/src/mongo/db/storage/rocks/rocks_init.cpp
index ad0fe429c78..aa696b6d900 100644
--- a/src/mongo/db/storage/rocks/rocks_init.cpp
+++ b/src/mongo/db/storage/rocks/rocks_init.cpp
@@ -43,7 +43,10 @@ namespace mongo {
         public:
             virtual ~RocksFactory(){}
             virtual StorageEngine* create( const StorageGlobalParams& params ) const {
-                return new KVStorageEngine(new RocksEngine(params.dbpath));
+                KVStorageEngineOptions options;
+                options.directoryPerDB = params.directoryperdb;
+                options.forRepair = params.repair;
+                return new KVStorageEngine(new RocksEngine(params.dbpath), options);
             }
 
             virtual StringData getCanonicalName() const {
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 7de2681019c..7d185b8af54 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -154,10 +154,16 @@ namespace mongo {
          */
         virtual int flushAllFiles( bool sync ) = 0;
 
-        virtual Status repairDatabase( OperationContext* txn,
-                                       const std::string& dbName,
-                                       bool preserveClonedFilesOnFailure = false,
-                                       bool backupOriginalFiles = false ) = 0;
+        /**
+         * Recover as much data as possible from a potentially corrupt RecordStore.
+         * This only recovers the record data, not indexes or anything else.
+         *
+         * Generally, this method should not be called directly except by the repairDatabase()
+         * free function.
+         *
+         * NOTE: MMAPv1 does not support this method and has its own repairDatabase() method.
+         */
+        virtual Status repairRecordStore(OperationContext* txn, const std::string& ns) = 0;
 
         /**
          * This method will be called before there is a clean shutdown.  Storage engines should
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
index 9578c30d284..5eab18c15c5 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
@@ -57,7 +57,8 @@ namespace mongo {
             virtual StorageEngine* create( const StorageGlobalParams& params ) const {
                 WiredTigerKVEngine* kv = new WiredTigerKVEngine( params.dbpath,
                                                                  wiredTigerGlobalOptions.engineConfig,
-                                                                 params.dur );
+                                                                 params.dur,
+                                                                 params.repair );
                 kv->setRecordStoreExtraOptions( wiredTigerGlobalOptions.collectionConfig );
                 kv->setSortedDataInterfaceExtraOptions( wiredTigerGlobalOptions.indexConfig );
                 // Intentionally leaked.
@@ -66,8 +67,8 @@ namespace mongo {
 
                 KVStorageEngineOptions options;
                 options.directoryPerDB = params.directoryperdb;
-                options.directoryForIndexes =
-                    wiredTigerGlobalOptions.directoryForIndexes;
+                options.directoryForIndexes = wiredTigerGlobalOptions.directoryForIndexes;
+                options.forRepair = params.repair;
                 return new KVStorageEngine( kv, options );
             }
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 2b429a5250f..eb4dabad796 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -46,6 +46,7 @@
 #include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
 #include "mongo/util/log.h"
 #include "mongo/util/processinfo.h"
+#include "mongo/util/scopeguard.h"
 
 namespace mongo {
 
@@ -89,12 +90,12 @@ namespace mongo {
                               WT_CURSOR *cursor) {
             return 0;
         }
-
     }
 
     WiredTigerKVEngine::WiredTigerKVEngine( const std::string& path,
                                             const std::string& extraOpenOptions,
-                                            bool durable )
+                                            bool durable,
+                                            bool repair )
         : _path( path ),
           _durable( durable ),
           _epoch( 0 ),
@@ -157,6 +158,10 @@ namespace mongo {
         _sizeStorerUri = "table:sizeStorer";
         {
             WiredTigerSession session( _conn, -1 );
+            if (repair && _hasUri(session.getSession(), _sizeStorerUri)) {
+                log() << "Repairing size cache";
+                fassertNoTrace(28577, _salvageIfNeeded(_sizeStorerUri.c_str()));
+            }
             WiredTigerSizeStorer* ss = new WiredTigerSizeStorer();
             ss->loadFrom( &session, _sizeStorerUri );
             _sizeStorer.reset( ss );
@@ -208,10 +213,26 @@ namespace mongo {
 
     Status WiredTigerKVEngine::repairIdent( OperationContext* opCtx,
                                             const StringData& ident ) {
-        WiredTigerSession session( _conn, -1 );
-        WT_SESSION* s = session.getSession();
+        WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession();
+        session->closeAllCursors();
         string uri = _uri(ident);
-        return wtRCToStatus( s->compact(s, uri.c_str(), NULL ) );
+        return _salvageIfNeeded(uri.c_str());
+    }
+
+    Status WiredTigerKVEngine::_salvageIfNeeded(const char* uri) {
+        // Using a side session to avoid transactional issues
+        WiredTigerSession* sessionWrapper = _sessionCache->getSession();
+        ON_BLOCK_EXIT(&WiredTigerSessionCache::releaseSession, _sessionCache.get(), sessionWrapper);
+        WT_SESSION* session = sessionWrapper->getSession();
+
+        if ((session->verify)(session, uri, NULL) == 0) {
+            log() << "Verify succeeded on uri " << uri << ". Not salvaging.";
+            return Status::OK();
+        }
+
+        // TODO need to cleanup the sizeStorer cache after salvaging.
+        log() << "Verify failed on uri " << uri << ". Running a salvage operation.";
+        return wtRCToStatus(session->salvage(session, uri, NULL), "Salvage failed:");
     }
 
     int WiredTigerKVEngine::flushAllFiles( bool sync ) {
@@ -400,6 +421,22 @@ namespace mongo {
         return true;
     }
 
+    bool WiredTigerKVEngine::hasIdent(OperationContext* opCtx, const StringData& ident) const {
+        return _hasUri(WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession(), _uri(ident));
+    }
+
+    bool WiredTigerKVEngine::_hasUri(WT_SESSION* session, const std::string& uri) const {
+        // can't use WiredTigerCursor since this is called from constructor.
+        WT_CURSOR* c = NULL;
+        int ret = session->open_cursor(session, "metadata:", NULL, NULL, &c);
+        if (ret == ENOENT) return false;
+        invariantWTOK(ret);
+        ON_BLOCK_EXIT(c->close, c);
+
+        c->set_key(c, uri.c_str());
+        return c->search(c) == 0;
+    }
+
     std::vector<std::string> WiredTigerKVEngine::getAllIdents( OperationContext* opCtx ) const {
         std::vector<std::string> all;
         WiredTigerCursor cursor( "metadata:", WiredTigerSession::kMetadataCursorId, opCtx );
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index e8f6b9e148e..8adf077658f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -52,7 +52,8 @@ namespace mongo {
     public:
         WiredTigerKVEngine( const std::string& path,
                             const std::string& extraOpenOptions = "",
-                            bool durable = true );
+                            bool durable = true,
+                            bool repair = false );
         virtual ~WiredTigerKVEngine();
 
         void setRecordStoreExtraOptions( const std::string& options );
@@ -99,6 +100,8 @@ namespace mongo {
         virtual Status repairIdent( OperationContext* opCtx,
                                     const StringData& ident );
 
+        virtual bool hasIdent(OperationContext* opCtx, const StringData& ident) const;
+
         std::vector<std::string> getAllIdents( OperationContext* opCtx ) const;
 
         virtual void cleanShutdown();
@@ -118,8 +121,11 @@ namespace mongo {
 
     private:
 
+        Status _salvageIfNeeded(const char* uri);
         void _checkIdentPath( const StringData& ident );
 
+        bool _hasUri(WT_SESSION* session, const std::string& uri) const;
+
         string _uri( const StringData& ident ) const;
         bool _drop( const StringData& ident );
 
diff --git a/src/mongo/util/assert_util.h b/src/mongo/util/assert_util.h
index 902172496cd..d2e750c3c99 100644
--- a/src/mongo/util/assert_util.h
+++ b/src/mongo/util/assert_util.h
@@ -305,7 +305,7 @@ namespace mongo {
 
 #ifdef MONGO_EXPOSE_MACROS
 # define dassert MONGO_dassert
-# define verify MONGO_verify
+# define verify(expression) MONGO_verify(expression)
 # define invariant MONGO_invariant
 # define invariantOK MONGO_invariantOK
 # define uassert MONGO_uassert