Better repair for WT

* Doesn't construct RecordStores or indexes before they've been repaired
* No longer need to skip checking index versions.
* Updates numRecords and dataSize after the repair.

Related issues:
SERVER-16817 Skip checking index versions in WT during --repair
SERVER-16172 --repair fails before repairing collections in WT

A call to flushAllFiles is commented out due to SERVER-16869. Resolving it
should uncomment that line.

19 files changed, 131 insertions, 76 deletions

diff --git a/src/mongo/db/catalog/collection.cpp b/src/mongo/db/catalog/collection.cpp
index 84a6f0465f9..4a7f7bf7947 100644
--- a/src/mongo/db/catalog/collection.cpp
+++ b/src/mongo/db/catalog/collection.cpp
@@ -42,7 +42,7 @@
 #include "mongo/db/commands/server_status_metric.h"
 #include "mongo/db/curop.h"
 #include "mongo/db/catalog/collection_catalog_entry.h"
-#include "mongo/db/catalog/database.h"
+#include "mongo/db/catalog/database_catalog_entry.h"
 #include "mongo/db/catalog/index_create.h"
 #include "mongo/db/index/index_access_method.h"
 #include "mongo/db/operation_context.h"
@@ -87,11 +87,11 @@ namespace mongo {
                             const StringData& fullNS,
                             CollectionCatalogEntry* details,
                             RecordStore* recordStore,
-                            Database* database )
+                            DatabaseCatalogEntry* dbce )
         : _ns( fullNS ),
           _details( details ),
           _recordStore( recordStore ),
-          _database( database ),
+          _dbce( dbce ),
           _infoCache( this ),
           _indexCatalog( this ),
           _cursorManager( fullNS ) {
diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h
index 7edf1064d1f..15a3e791953 100644
--- a/src/mongo/db/catalog/collection.h
+++ b/src/mongo/db/catalog/collection.h
@@ -47,7 +47,7 @@
 namespace mongo {
 
     class CollectionCatalogEntry;
-    class Database;
+    class DatabaseCatalogEntry;
     class ExtentManager;
     class IndexCatalog;
     class MultiIndexBlock;
@@ -105,7 +105,7 @@ namespace mongo {
                     const StringData& fullNS,
                     CollectionCatalogEntry* details, // does not own
                     RecordStore* recordStore, // does not own
-                    Database* database ); // does not own
+                    DatabaseCatalogEntry* dbce ); // does not own
 
         ~Collection();
 
@@ -308,7 +308,7 @@ namespace mongo {
         NamespaceString _ns;
         CollectionCatalogEntry* _details;
         RecordStore* _recordStore;
-        Database* _database;
+        DatabaseCatalogEntry* _dbce;
         CollectionInfoCache _infoCache;
         IndexCatalog _indexCatalog;
 
diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp
index 3028f5e4f59..c7aa8012d44 100644
--- a/src/mongo/db/catalog/database.cpp
+++ b/src/mongo/db/catalog/database.cpp
@@ -185,7 +185,7 @@ namespace mongo {
         invariant( rs.get() ); // if cce exists, so should this
 
         // Not registering AddCollectionChange since this is for collections that already exist.
-        Collection* c = new Collection( txn, fullns, cce.release(), rs.release(), this );
+        Collection* c = new Collection( txn, fullns, cce.release(), rs.release(), _dbEntry );
         return c;
     }
 
diff --git a/src/mongo/db/catalog/index_catalog.cpp b/src/mongo/db/catalog/index_catalog.cpp
index aff072ff221..27ba3644cc6 100644
--- a/src/mongo/db/catalog/index_catalog.cpp
+++ b/src/mongo/db/catalog/index_catalog.cpp
@@ -135,9 +135,9 @@ namespace mongo {
                                                                   descriptorCleanup.release(),
                                                                   _collection->infoCache() ) );
 
-        entry->init( txn, _collection->_database->_dbEntry->getIndex( txn,
-                                                                      _collection->getCatalogEntry(),
-                                                                      entry.get() ) );
+        entry->init( txn, _collection->_dbce->getIndex( txn,
+                                                        _collection->getCatalogEntry(),
+                                                        entry.get() ) );
 
         IndexCatalogEntry* save = entry.get();
         _entries.add( entry.release() );
@@ -175,7 +175,7 @@ namespace mongo {
         string pluginName = IndexNames::findPluginName(keyPattern);
         bool known = IndexNames::isKnownName(pluginName);
 
-        if ( !_collection->_database->getDatabaseCatalogEntry()->isOlderThan24( txn ) ) {
+        if ( !_collection->_dbce->isOlderThan24( txn ) ) {
             // RulesFor24+
             // This assert will be triggered when downgrading from a future version that
             // supports an index plugin unsupported by this version.
@@ -222,20 +222,21 @@ namespace mongo {
             return Status::OK();
         }
 
-        Database* db = _collection->_database;
+        DatabaseCatalogEntry* dbce = _collection->_dbce;
 
-        if ( !db->getDatabaseCatalogEntry()->isOlderThan24( txn ) ) {
+        if ( !dbce->isOlderThan24( txn ) ) {
             return Status::OK(); // these checks have already been done
         }
 
-        auto_ptr<PlanExecutor> exec(
-            InternalPlanner::collectionScan(txn,
-                                            db->_indexesName,
-                                            db->getCollection(db->_indexesName)));
+        // Everything below is MMAPv1 specific since it was the only storage engine that existed
+        // before 2.4. We look at all indexes in this database to make sure that none of them use
+        // plugins that didn't exist before 2.4. If that holds, we mark the database as "2.4-clean"
+        // which allows creation of indexes using new plugins.
 
-        BSONObj index;
-        PlanExecutor::ExecState state;
-        while ( PlanExecutor::ADVANCED == (state = exec->getNext(&index, NULL)) ) {
+        RecordStore* indexes = dbce->getRecordStore(dbce->name() + ".system.indexes");
+        boost::scoped_ptr<RecordIterator> it(indexes->getIterator(txn));
+        while (!it->isEOF()) {
+            const BSONObj index = it->dataFor(it->getNext()).toBson();
             const BSONObj key = index.getObjectField("key");
             const string plugin = IndexNames::findPluginName(key);
             if ( IndexNames::existedBefore24(plugin) )
@@ -250,11 +251,7 @@ namespace mongo {
             return Status( ErrorCodes::CannotCreateIndex, errmsg );
         }
 
-        if ( PlanExecutor::IS_EOF != state ) {
-            warning() << "Internal error while reading system.indexes collection";
-        }
-
-        db->_dbEntry->markIndexSafe24AndUp( txn );
+        dbce->markIndexSafe24AndUp( txn );
 
         return Status::OK();
     }
@@ -837,15 +834,9 @@ namespace {
 
             log() << "error dropping index: " << indexNamespace
                   << " going to leak some memory to be safe";
-
-
-            _collection->_database->_clearCollectionCache( txn, indexNamespace );
-
             throw;
         }
 
-        _collection->_database->_clearCollectionCache( txn, indexNamespace );
-
         _checkMagic();
 
         return Status::OK();
diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp
index 26f4dc5382e..516827a3715 100644
--- a/src/mongo/db/repair_database.cpp
+++ b/src/mongo/db/repair_database.cpp
@@ -84,8 +84,7 @@ namespace {
         // Skip the rest if there are no indexes to rebuild.
         if (indexSpecs.empty()) return Status::OK();
 
-        boost::scoped_ptr<Database> db;
-        Collection* collection;
+        boost::scoped_ptr<Collection> collection;
         boost::scoped_ptr<MultiIndexBlock> indexer;
         {
             // These steps are combined into a single WUOW to ensure there are no commits without
@@ -106,11 +105,10 @@ namespace {
             // Indexes must be dropped before we open the Collection otherwise we could attempt to
             // open a bad index and fail.
             // TODO see if MultiIndexBlock can be made to work without a Collection.
-            db.reset(new Database(txn, dbce->name(), dbce));
-            collection = db->getCollection(collectionName);
-            invariant(collection);
+            const StringData ns = cce->ns().ns();
+            collection.reset(new Collection(txn, ns, cce, dbce->getRecordStore(ns), dbce));
 
-            indexer.reset(new MultiIndexBlock(txn, collection));
+            indexer.reset(new MultiIndexBlock(txn, collection.get()));
             Status status = indexer->init(indexSpecs);
             if (!status.isOK()) {
                 // The WUOW will handle cleanup, so the indexer shouldn't do its own.
@@ -124,6 +122,9 @@ namespace {
         // Iterate all records in the collection. Delete them if they aren't valid BSON. Index them
         // if they are.
 
+        long long numRecords = 0;
+        long long dataSize = 0;
+
         RecordStore* rs = collection->getRecordStore();
         boost::scoped_ptr<RecordIterator> it(rs->getIterator(txn));
         while (!it->isEOF()) {
@@ -144,6 +145,9 @@ namespace {
                 continue;
             }
 
+            numRecords++;
+            dataSize += data.size();
+
             // Now index the record.
             // TODO SERVER-14812 add a mode that drops duplicates rather than failing
             WriteUnitOfWork wunit(txn);
@@ -158,6 +162,7 @@ namespace {
         {
             WriteUnitOfWork wunit(txn);
             indexer->commit();
+            rs->updateStatsAfterRepair(txn, numRecords, dataSize);
             wunit.commit();
         }
 
@@ -232,6 +237,9 @@ namespace {
 
             status = rebuildIndexesOnCollection(txn, dbce, *it);
             if (!status.isOK()) return status;
+
+            // TODO: uncomment once SERVER-16869
+            // engine->flushAllFiles(true);
         }
 
         return Status::OK();
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index 7d3cc83e666..768688d3569 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -170,6 +170,11 @@ namespace mongo {
             return Status::OK();
         }
 
+        virtual void updateStatsAfterRepair(OperationContext* txn,
+                                            long long numRecords,
+                                            long long dataSize) {
+        }
+
     private:
         CollectionOptions _options;
         long long _numInserts;
diff --git a/src/mongo/db/storage/in_memory/in_memory_record_store.h b/src/mongo/db/storage/in_memory/in_memory_record_store.h
index be5bb9191aa..c371a12135f 100644
--- a/src/mongo/db/storage/in_memory/in_memory_record_store.h
+++ b/src/mongo/db/storage/in_memory/in_memory_record_store.h
@@ -136,6 +136,13 @@ namespace mongo {
         virtual boost::optional<RecordId> oplogStartHack(OperationContext* txn,
                                                          const RecordId& startingPosition) const;
 
+        virtual void updateStatsAfterRepair(OperationContext* txn,
+                                            long long numRecords,
+                                            long long dataSize) {
+            invariant(_data->records.size() == size_t(numRecords));
+            _data->dataSize = dataSize;
+        }
+
     protected:
         struct InMemoryRecord {
             InMemoryRecord() :size(0) {}
diff --git a/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp
index f397c767dca..456ce30531f 100644
--- a/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp
+++ b/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp
@@ -233,14 +233,23 @@ namespace mongo {
     }
 
     void KVDatabaseCatalogEntry::initCollection( OperationContext* opCtx,
-                                                 const std::string& ns ) {
-        string ident = _engine->getCatalog()->getCollectionIdent( ns );
-        BSONCollectionCatalogEntry::MetaData md = _engine->getCatalog()->getMetaData( opCtx, ns );
+                                                 const std::string& ns,
+                                                 bool forRepair ) {
+        invariant(!_collections.count(ns));
 
-        RecordStore* rs = _engine->getEngine()->getRecordStore( opCtx, ns, ident, md.options );
-        invariant( rs );
+        const std::string ident = _engine->getCatalog()->getCollectionIdent( ns );
 
-        invariant(!_collections.count(ns));
+        RecordStore* rs;
+        if (forRepair) {
+            // Using a NULL rs since we don't want to open this record store before it has been
+            // repaired. This also ensures that if we try to use it, it will blow up.
+            rs = NULL;
+        }
+        else {
+            BSONCollectionCatalogEntry::MetaData md = _engine->getCatalog()->getMetaData(opCtx, ns);
+            rs = _engine->getEngine()->getRecordStore( opCtx, ns, ident, md.options );
+            invariant( rs );
+        }
 
         // No change registration since this is only for committed collections
         _collections[ns] = new KVCollectionCatalogEntry( _engine->getEngine(),
@@ -250,6 +259,18 @@ namespace mongo {
                                                          rs );
     }
 
+    void KVDatabaseCatalogEntry::reinitCollectionAfterRepair(OperationContext* opCtx,
+                                                             const std::string& ns) {
+        // Get rid of the old entry.
+        CollectionMap::iterator it = _collections.find(ns);
+        invariant(it != _collections.end());
+        delete it->second;
+        _collections.erase(it);
+
+        // Now reopen fully initialized.
+        initCollection(opCtx, ns, false);
+    }
+
     Status KVDatabaseCatalogEntry::renameCollection( OperationContext* txn,
                                                      const StringData& fromNS,
                                                      const StringData& toNS,
diff --git a/src/mongo/db/storage/kv/kv_database_catalog_entry.h b/src/mongo/db/storage/kv/kv_database_catalog_entry.h
index fc5fb39adbd..4cb5d1d3edc 100644
--- a/src/mongo/db/storage/kv/kv_database_catalog_entry.h
+++ b/src/mongo/db/storage/kv/kv_database_catalog_entry.h
@@ -86,7 +86,11 @@ namespace mongo {
         // --------------
 
         void initCollection( OperationContext* opCtx,
-                             const std::string& ns );
+                             const std::string& ns,
+                             bool forRepair );
+
+        void initCollectionBeforeRepair(OperationContext* opCtx, const std::string& ns);
+        void reinitCollectionAfterRepair(OperationContext* opCtx, const std::string& ns);
 
     private:
         class AddCollectionChange;
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp
index ef56ed40325..fc2ddbb580b 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp
@@ -119,7 +119,8 @@ namespace mongo {
                 if ( !db ) {
                     db = new KVDatabaseCatalogEntry( dbName, this );
                 }
-                db->initCollection( &opCtx, coll );
+
+                db->initCollection( &opCtx, coll, options.forRepair );
             }
 
             uow.commit();
@@ -262,6 +263,11 @@ namespace mongo {
     }
 
     Status KVStorageEngine::repairRecordStore(OperationContext* txn, const std::string& ns) {
-        return _engine->repairIdent(txn, _catalog->getCollectionIdent(ns));
+        Status status = _engine->repairIdent(txn, _catalog->getCollectionIdent(ns));
+        if (!status.isOK())
+            return status;
+
+        _dbs[nsToDatabase(ns)]->reinitCollectionAfterRepair(txn, ns);
+        return Status::OK();
     }
 }
diff --git a/src/mongo/db/storage/mmap_v1/heap_record_store_btree.h b/src/mongo/db/storage/mmap_v1/heap_record_store_btree.h
index cbbc17b2d34..4553620c6d9 100644
--- a/src/mongo/db/storage/mmap_v1/heap_record_store_btree.h
+++ b/src/mongo/db/storage/mmap_v1/heap_record_store_btree.h
@@ -161,6 +161,12 @@ namespace mongo {
         virtual bool isCapped() const { invariant(false); }
 
         virtual const char* name() const { invariant(false); }
+
+        virtual void updateStatsAfterRepair(OperationContext* txn,
+                                            long long numRecords,
+                                            long long dataSize) {
+            invariant(false);
+        }
         // more things that we actually care about below
 
     private:
diff --git a/src/mongo/db/storage/mmap_v1/record_store_v1_base.h b/src/mongo/db/storage/mmap_v1/record_store_v1_base.h
index b67f68a2cea..ec649461814 100644
--- a/src/mongo/db/storage/mmap_v1/record_store_v1_base.h
+++ b/src/mongo/db/storage/mmap_v1/record_store_v1_base.h
@@ -262,6 +262,12 @@ namespace mongo {
         virtual Status setCustomOption( OperationContext* txn,
                                         const BSONElement& option,
                                         BSONObjBuilder* info = NULL );
+
+        virtual void updateStatsAfterRepair(OperationContext* txn,
+                                            long long numRecords,
+                                            long long dataSize) {
+            invariant(false); // MMAPv1 has its own repair which doesn't call this.
+        }
     protected:
 
         virtual Record* recordFor( const DiskLoc& loc ) const;
diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h
index 0896579e5a4..c94061d1701 100644
--- a/src/mongo/db/storage/record_store.h
+++ b/src/mongo/db/storage/record_store.h
@@ -331,6 +331,12 @@ namespace mongo {
             return Status::OK();
         }
 
+        /**
+         * Called after a repair operation is run with the recomputed numRecords and dataSize.
+         */
+        virtual void updateStatsAfterRepair(OperationContext* txn,
+                                            long long numRecords,
+                                            long long dataSize) = 0;
 
     protected:
         std::string _ns;
diff --git a/src/mongo/db/storage/rocks/rocks_record_store.h b/src/mongo/db/storage/rocks/rocks_record_store.h
index 2406837f0fa..9aa951bc764 100644
--- a/src/mongo/db/storage/rocks/rocks_record_store.h
+++ b/src/mongo/db/storage/rocks/rocks_record_store.h
@@ -172,6 +172,12 @@ namespace mongo {
 
         virtual Status oplogDiskLocRegister(OperationContext* txn, const OpTime& opTime);
 
+        virtual void updateStatsAfterRepair(OperationContext* txn,
+                                            long long numRecords,
+                                            long long dataSize) {
+            // TODO
+        }
+
         void setCappedDeleteCallback(CappedDocumentDeleteCallback* cb) {
           _cappedDeleteCallback = cb;
         }
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index 82d77b08746..ca5c793428f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -67,8 +67,6 @@ namespace {
 
     static const WiredTigerItem emptyItem(NULL, 0);
 
-    bool shouldCheckIndexVersions = true;
-
     static const int kMinimumIndexVersion = 6;
     static const int kCurrentIndexVersion = 6; // New indexes use this by default.
     static const int kMaximumIndexVersion = 6;
@@ -115,11 +113,6 @@ namespace {
 } // namespace
 
     // static
-    void WiredTigerIndex::disableVersionCheckForRepair() {
-        shouldCheckIndexVersions = false;
-    }
-
-    // static
     StatusWith<std::string> WiredTigerIndex::parseIndexOptions(const BSONObj& options) {
         BSONForEach(elem, options) {
             if (elem.fieldNameStringData() == "configString") {
@@ -209,17 +202,14 @@ namespace {
           _uri( uri ),
           _instanceId( WiredTigerSession::genCursorId() ) {
 
-        if (shouldCheckIndexVersions) {
-            Status versionStatus =
-                WiredTigerUtil::checkApplicationMetadataFormatVersion(ctx,
-                                                                      uri,
-                                                                      kMinimumIndexVersion,
-                                                                      kMaximumIndexVersion);
-            if (!versionStatus.isOK()) {
-                fassertFailedWithStatusNoTrace(28579, versionStatus);
-            }
+        Status versionStatus =
+            WiredTigerUtil::checkApplicationMetadataFormatVersion(ctx,
+                                                                  uri,
+                                                                  kMinimumIndexVersion,
+                                                                  kMaximumIndexVersion);
+        if (!versionStatus.isOK()) {
+            fassertFailedWithStatusNoTrace(28579, versionStatus);
         }
-
     }
 
     Status WiredTigerIndex::insert(OperationContext* txn,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h
index cf4ba3c0c45..81f6aa57c1f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h
@@ -113,14 +113,6 @@ namespace mongo {
 
         virtual bool unique() const = 0;
 
-        /**
-         * Call this during engine initialization to disable index version checking.
-         *
-         * WARNING: this should only be called when started with --repair. It is safe then because
-         * we drop existing indexes before attempting to use them.
-         */
-        static void disableVersionCheckForRepair();
-
     protected:
 
         virtual Status _insert( WT_CURSOR* c,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 84543d3d439..040429f6d98 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -108,11 +108,6 @@ namespace mongo {
           _durable( durable ),
           _sizeStorerSyncTracker( 100000, 60 * 1000 ) {
 
-        if (repair) {
-            // This should be done once before we try to access any data.
-            WiredTigerIndex::disableVersionCheckForRepair();
-        }
-
         _eventHandler.handle_error = mdb_handle_error;
         _eventHandler.handle_message = mdb_handle_message;
         _eventHandler.handle_progress = mdb_handle_progress;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index 5e2bde239cf..c399694c051 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -856,6 +856,14 @@ namespace {
         return _fromKey(key);
     }
 
+    void WiredTigerRecordStore::updateStatsAfterRepair(OperationContext* txn,
+                                                       long long numRecords,
+                                                       long long dataSize) {
+        _numRecords.store(numRecords);
+        _dataSize.store(dataSize);
+        _sizeStorer->store(_uri, numRecords, dataSize);
+    }
+
     RecordId WiredTigerRecordStore::_nextId() {
         invariant(!_useOplogHack);
         RecordId out = RecordId(_nextIdNum.fetchAndAdd(1));
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h
index e363090e1d0..74dc4ec30bd 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h
@@ -173,6 +173,10 @@ namespace mongo {
         virtual Status oplogDiskLocRegister( OperationContext* txn,
                                              const OpTime& opTime );
 
+        virtual void updateStatsAfterRepair(OperationContext* txn,
+                                            long long numRecords,
+                                            long long dataSize);
+
         bool isOplog() const { return _isOplog; }
         bool usingOplogHack() const { return _useOplogHack; }