SERVER-11178: index building now all with IndexCatalog, CatalogHack out of all new paths

13 files changed, 212 insertions, 121 deletions

diff --git a/src/mongo/db/catalog/index_catalog.cpp b/src/mongo/db/catalog/index_catalog.cpp
index 1652d1601b1..4030f22fa8f 100644
--- a/src/mongo/db/catalog/index_catalog.cpp
+++ b/src/mongo/db/catalog/index_catalog.cpp
@@ -227,12 +227,31 @@ namespace mongo {
         verify( indexBuildBlock.indexDetails() );
 
         try {
-            buildAnIndex( _collection,
-                          *indexBuildBlock.indexDetails(), mayInterrupt);
+            int idxNo = _details->findIndexByName( idxName, true );
+            verify( idxNo >= 0 );
+
+            IndexDetails* id = &_details->idx(idxNo);
+
+            scoped_ptr<IndexDescriptor> desc( new IndexDescriptor( _collection, idxNo,
+                                                                   id, id->info.obj().getOwned() ) );
+            buildAnIndex( _collection, desc.get(), mayInterrupt );
             indexBuildBlock.success();
+
+            // in case we got any access methods or something like that
+            // TEMP until IndexDescriptor has to direct refs
+            idxNo = _details->findIndexByName( idxName, true );
+            verify( idxNo >= 0 );
+            _deleteCacheEntry( idxNo );
+
             return Status::OK();
         }
         catch (DBException& e) {
+            // in case we got any access methods or something like that
+            // TEMP until IndexDescriptor has to direct refs
+            int idxNo = _details->findIndexByName( idxName, true );
+            verify( idxNo >= 0 );
+            _deleteCacheEntry( idxNo );
+
             // save our error msg string as an exception or dropIndexes will overwrite our message
             LastError *le = lastError.get();
             int savecode = 0;
@@ -312,6 +331,8 @@ namespace mongo {
     void IndexCatalog::IndexBuildBlock::success() {
 
         fassert( 17206, _indexDetails );
+        BSONObj keyPattern = _indexDetails->keyPattern().getOwned();
+
         _indexDetails = NULL;
 
         fassert( 17207, _catalog->_collection->ok() );
@@ -326,7 +347,7 @@ namespace mongo {
 
             int toIdxNo = _nsd->getCompletedIndexCount();
 
-            _nsd->swapIndex( _ns.c_str(), idxNo, toIdxNo );
+            _nsd->swapIndex( idxNo, toIdxNo );
 
             // neither of these should be used in queries yet, so nothing should be caching these
             _catalog->_deleteCacheEntry( idxNo );
@@ -342,7 +363,7 @@ namespace mongo {
 
         _catalog->_fixDescriptorCacheNumbers();
 
-        IndexLegacy::postBuildHook( _catalog->_collection, _catalog->getDescriptor( idxNo ) );
+        IndexLegacy::postBuildHook( _catalog->_collection, keyPattern );
     }
 
 
@@ -691,6 +712,11 @@ namespace mongo {
         return Status::OK();
     }
 
+    void IndexCatalog::markMultikey( IndexDescriptor* idx, bool isMultikey ) {
+        if ( _details->setIndexIsMultikey( idx->_indexNumber, isMultikey ) )
+            _collection->infoCache()->clearQueryCache();
+    }
+
     // ---------------------------
 
     int IndexCatalog::numIndexesTotal() const {
@@ -710,15 +736,17 @@ namespace mongo {
         return NULL;
     }
 
-    IndexDescriptor* IndexCatalog::findIndexByName( const StringData& name ) {
-        int idxNo = _details->findIndexByName( name );
+    IndexDescriptor* IndexCatalog::findIndexByName( const StringData& name,
+                                                    bool includeUnfinishedIndexes ) {
+        int idxNo = _details->findIndexByName( name, includeUnfinishedIndexes );
         if ( idxNo < 0 )
             return NULL;
         return getDescriptor( idxNo );
     }
 
-    IndexDescriptor* IndexCatalog::findIndexByKeyPattern( const BSONObj& key ) {
-        int idxNo = _details->findIndexByKeyPattern( key );
+    IndexDescriptor* IndexCatalog::findIndexByKeyPattern( const BSONObj& key,
+                                                          bool includeUnfinishedIndexes ) {
+        int idxNo = _details->findIndexByKeyPattern( key, includeUnfinishedIndexes );
         if ( idxNo < 0 )
             return NULL;
         return getDescriptor( idxNo );
@@ -774,6 +802,25 @@ namespace mongo {
         return newlyCreated;
     }
 
+    BtreeBasedAccessMethod* IndexCatalog::getBtreeBasedIndex( IndexDescriptor* desc ) {
+
+        string type = _getAccessMethodName(desc->keyPattern());
+
+        if (IndexNames::HASHED == type ||
+            IndexNames::GEO_2DSPHERE == type ||
+            IndexNames::TEXT == type || IndexNames::TEXT_INTERNAL == type ||
+            IndexNames::GEO_HAYSTACK == type ||
+            "" == type ||
+            IndexNames::GEO_2D == type ) {
+            IndexAccessMethod* iam = getIndex( desc );
+            return dynamic_cast<BtreeBasedAccessMethod*>( iam );
+        }
+
+        error() << "getBtreeBasedIndex with a non btree index (" << type << ")";
+        verify(0);
+        return NULL;
+    }
+
 
     IndexAccessMethod* IndexCatalog::getIndex( IndexDescriptor* desc ) {
         _checkMagic();
diff --git a/src/mongo/db/catalog/index_catalog.h b/src/mongo/db/catalog/index_catalog.h
index d500a0747c6..03bfe1b1be6 100644
--- a/src/mongo/db/catalog/index_catalog.h
+++ b/src/mongo/db/catalog/index_catalog.h
@@ -71,12 +71,14 @@ namespace mongo {
         /**
          * @return null if cannot find
          */
-        IndexDescriptor* findIndexByName( const StringData& name );
+        IndexDescriptor* findIndexByName( const StringData& name,
+                                          bool includeUnfinishedIndexes = false );
 
         /**
          * @return null if cannot find
          */
-        IndexDescriptor* findIndexByKeyPattern( const BSONObj& key );
+        IndexDescriptor* findIndexByKeyPattern( const BSONObj& key,
+                                                bool includeUnfinishedIndexes = false );
 
         /* Returns the index entry for the first index whose prefix contains
          * 'keyPattern'. If 'requireSingleKey' is true, skip indices that contain
@@ -126,6 +128,8 @@ namespace mongo {
          */
         BSONObj prepOneUnfinishedIndex();
 
+        void markMultikey( IndexDescriptor* idx, bool isMultikey = true );
+
         // --- these probably become private?
 
         class IndexBuildBlock {
diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp
index a5e7bc077da..b64d5d9922e 100644
--- a/src/mongo/db/catalog/index_create.cpp
+++ b/src/mongo/db/catalog/index_create.cpp
@@ -55,21 +55,16 @@ namespace mongo {
     /**
      * Add the provided (obj, dl) pair to the provided index.
      */
-    static void addKeysToIndex( Collection* collection, int idxNo,
+    static void addKeysToIndex( Collection* collection, IndexDescriptor* desc,
                                 const BSONObj& obj, const DiskLoc &recordLoc ) {
 
-        IndexDetails& id = collection->details()->idx(idxNo);
-
-        IndexDescriptor* desc = collection->getIndexCatalog()->getDescriptor( idxNo );
         verify( desc );
-
         IndexAccessMethod* iam = collection->getIndexCatalog()->getIndex( desc );
-        verify( iam );
 
         InsertDeleteOptions options;
         options.logIfError = false;
-        options.dupsAllowed = (!KeyPattern::isIdKeyPattern(id.keyPattern()) && !id.unique())
-            || ignoreUniqueIndex(id);
+        options.dupsAllowed = (!KeyPattern::isIdKeyPattern(desc->keyPattern()) && !desc->unique())
+            || ignoreUniqueIndex(desc);
 
         int64_t inserted;
         Status ret = iam->insert(obj, recordLoc, options, &inserted);
@@ -87,11 +82,11 @@ namespace mongo {
             : BackgroundOperation(ns) {
         }
 
-        unsigned long long go( Collection* collection, IndexDetails& idx );
+        unsigned long long go( Collection* collection, IndexDescriptor* idx );
 
     private:
         unsigned long long addExistingToIndex( Collection* collection,
-                                               IndexDetails& idx );
+                                               IndexDescriptor* idx );
 
         void prep(const StringData& ns ) {
             Lock::assertWriteLocked(ns);
@@ -108,7 +103,7 @@ namespace mongo {
 
     };
 
-    unsigned long long BackgroundIndexBuildJob::go( Collection* collection, IndexDetails& idx) {
+    unsigned long long BackgroundIndexBuildJob::go( Collection* collection, IndexDescriptor* idx) {
 
         string ns = collection->ns().ns();
 
@@ -119,7 +114,7 @@ namespace mongo {
         prep( ns );
 
         try {
-            idx.head.writing() = BtreeBasedBuilder::makeEmptyIndex( idx );
+            idx->getOnDisk().head.writing() = BtreeBasedBuilder::makeEmptyIndex( idx->getOnDisk() );
             unsigned long long n = addExistingToIndex( collection, idx );
             // idx may point at an invalid index entry at this point
             done( ns );
@@ -132,12 +127,12 @@ namespace mongo {
     }
 
     unsigned long long BackgroundIndexBuildJob::addExistingToIndex( Collection* collection,
-                                                                    IndexDetails& idx ) {
+                                                                    IndexDescriptor* idx ) {
 
         string ns = collection->ns().ns(); // our copy for sanity
 
-        bool dupsAllowed = !idx.unique();
-        bool dropDups = idx.dropDups();
+        bool dupsAllowed = !idx->unique();
+        bool dropDups = idx->dropDups();
 
         ProgressMeter& progress = cc().curop()->setMessage("bg index build",
                                                            "Background Index Build Progress",
@@ -152,9 +147,7 @@ namespace mongo {
         // happens.
         RunnerYieldPolicy yieldPolicy;
 
-        std::string idxName = idx.indexName();
-        int idxNo = collection->details()->findIndexByName( idxName, true );
-        verify( idxNo >= 0 );
+        std::string idxName = idx->indexName();
 
         // After this yields in the loop, idx may point at a different index (if indexes get
         // flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should
@@ -166,10 +159,10 @@ namespace mongo {
             try {
                 if ( !dupsAllowed && dropDups ) {
                     LastError::Disabled led( lastError.get() );
-                    addKeysToIndex(collection, idxNo, js, loc);
+                    addKeysToIndex(collection, idx, js, loc);
                 }
                 else {
-                    addKeysToIndex(collection, idxNo, js, loc);
+                    addKeysToIndex(collection, idx, js, loc);
                 }
             }
             catch( AssertionException& e ) {
@@ -218,8 +211,8 @@ namespace mongo {
 
                 progress.setTotalWhileRunning( collection->numRecords() );
                 // Recalculate idxNo if we yielded
-                idxNo = collection->details()->findIndexByName( idxName, true );
-                verify( idxNo >= 0 );
+                idx = collection->getIndexCatalog()->findIndexByName( idxName, true );
+                verify( idx );
             }
         }
 
@@ -233,17 +226,17 @@ namespace mongo {
 
     // throws DBException
     void buildAnIndex( Collection* collection,
-                       IndexDetails& idx,
+                       IndexDescriptor* idx,
                        bool mayInterrupt ) {
 
         string ns = collection->ns().ns(); // our copy
 
-        BSONObj idxInfo = idx.info.obj();
+        const BSONObj& idxInfo = idx->infoObj();
 
         MONGO_TLOG(0) << "build index on: " << ns
-                      << " properties: " << idxInfo.jsonString() << endl;
+                      << " properties: " << idx->toString() << endl;
 
-        audit::logCreateIndex( currentClient.get(), &idxInfo, idx.indexName(), ns );
+        audit::logCreateIndex( currentClient.get(), &idxInfo, idx->indexName(), ns );
 
         Timer t;
         unsigned long long n;
@@ -251,18 +244,15 @@ namespace mongo {
         verify( Lock::isWriteLocked( ns ) );
 
         if( inDBRepair || !idxInfo["background"].trueValue() ) {
-            int idxNo = collection->details()->findIndexByName( idx.info.obj()["name"].valuestr(),
-                                                                true );
-            verify( idxNo >= 0 );
-            n = BtreeBasedBuilder::fastBuildIndex( ns.c_str(), collection->details(),
-                                                   idx, mayInterrupt, idxNo );
-            verify( !idx.head.isNull() );
+            n = BtreeBasedBuilder::fastBuildIndex( collection, idx, mayInterrupt );
+            verify( !idx->getHead().isNull() );
         }
         else {
             BackgroundIndexBuildJob j( ns );
             n = j.go( collection, idx );
         }
-        MONGO_TLOG(0) << "build index done.  scanned " << n << " total records. " << t.millis() / 1000.0 << " secs" << endl;
+        MONGO_TLOG(0) << "build index done.  scanned " << n << " total records. "
+                      << t.millis() / 1000.0 << " secs" << endl;
     }
 
 }  // namespace mongo
diff --git a/src/mongo/db/catalog/index_create.h b/src/mongo/db/catalog/index_create.h
index 333169b342e..f28034ac18c 100644
--- a/src/mongo/db/catalog/index_create.h
+++ b/src/mongo/db/catalog/index_create.h
@@ -32,16 +32,15 @@
 
 #include <string>
 
-#include "mongo/db/storage/index_details.h"
-
 namespace mongo {
     class Collection;
+    class IndexDescriptor;
 
     // Build an index in the foreground
     // If background is false, uses fast index builder
     // If background is true, uses background index builder; blocks until done.
     void buildAnIndex( Collection* collection,
-                       IndexDetails& idx,
+                       IndexDescriptor* idx,
                        bool mayInterrupt );
 
 } // namespace mongo
diff --git a/src/mongo/db/index/btree_based_builder.cpp b/src/mongo/db/index/btree_based_builder.cpp
index e8137eef06d..6f89edaf019 100644
--- a/src/mongo/db/index/btree_based_builder.cpp
+++ b/src/mongo/db/index/btree_based_builder.cpp
@@ -36,6 +36,7 @@
 #include "mongo/db/pdfile_private.h"
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/repl/is_master.h"
+#include "mongo/db/repl/oplog.h"
 #include "mongo/db/repl/rs.h"
 #include "mongo/db/sort_phase_one.h"
 #include "mongo/util/processinfo.h"
@@ -74,7 +75,7 @@ namespace mongo {
 
     template< class V >
     void buildBottomUpPhases2And3( bool dupsAllowed,
-                                   IndexDetails& idx,
+                                   IndexDescriptor* idx,
                                    BSONObjExternalSorter& sorter,
                                    bool dropDups,
                                    set<DiskLoc>& dupsToDrop,
@@ -83,7 +84,7 @@ namespace mongo {
                                    ProgressMeterHolder& pm,
                                    Timer& t,
                                    bool mayInterrupt ) {
-        BtreeBuilder<V> btBuilder(dupsAllowed, idx);
+        BtreeBuilder<V> btBuilder(dupsAllowed, idx->getOnDisk());
         BSONObj keyLast;
         auto_ptr<BSONObjExternalSorter::Iterator> i = sorter.iterator();
         // verifies that pm and op refer to the same ProgressMeter
@@ -153,19 +154,21 @@ namespace mongo {
         }
     }
 
-    void BtreeBasedBuilder::addKeysToPhaseOne(NamespaceDetails* d, const char* ns,
-                           const IndexDetails& idx,
-                           const BSONObj& order,
-                           SortPhaseOne* phaseOne,
-                           int64_t nrecords,
-                           ProgressMeter* progressMeter,
-                           bool mayInterrupt, int idxNo) {
-        auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
-        phaseOne->sortCmp.reset(getComparison(idx.version(), idx.keyPattern()));
+    void BtreeBasedBuilder::addKeysToPhaseOne(Collection* collection,
+                                              IndexDescriptor* idx,
+                                              const BSONObj& order,
+                                              SortPhaseOne* phaseOne,
+                                              ProgressMeter* progressMeter,
+                                              bool mayInterrupt ) {
+
+
+        phaseOne->sortCmp.reset(getComparison(idx->version(), idx->keyPattern()));
         phaseOne->sorter.reset(new BSONObjExternalSorter(phaseOne->sortCmp.get()));
-        phaseOne->sorter->hintNumObjects( nrecords );
-        auto_ptr<IndexDescriptor> desc(CatalogHack::getDescriptor(d, idxNo));
-        auto_ptr<BtreeBasedAccessMethod> iam(CatalogHack::getBtreeBasedIndex(desc.get()));
+        phaseOne->sorter->hintNumObjects( collection->numRecords() );
+
+        BtreeBasedAccessMethod* iam =collection->getIndexCatalog()->getBtreeBasedIndex( idx );
+
+        auto_ptr<Runner> runner(InternalPlanner::collectionScan(collection->ns().ns()));
         BSONObj o;
         DiskLoc loc;
         Runner::RunnerState state;
@@ -185,20 +188,20 @@ namespace mongo {
 
     }
 
-    uint64_t BtreeBasedBuilder::fastBuildIndex(const char* ns, NamespaceDetails* d,
-                                               IndexDetails& idx, bool mayInterrupt,
-                                               int idxNo) {
+    uint64_t BtreeBasedBuilder::fastBuildIndex( Collection* collection,
+                                                IndexDescriptor* idx,
+                                                bool mayInterrupt ) {
         CurOp * op = cc().curop();
 
         Timer t;
 
-        MONGO_TLOG(1) << "fastBuildIndex " << ns << ' ' << idx.info.obj().toString() << endl;
+        MONGO_TLOG(1) << "fastBuildIndex " << collection->ns() << ' ' << idx->toString() << endl;
 
-        bool dupsAllowed = !idx.unique() || ignoreUniqueIndex(idx);
-        bool dropDups = idx.dropDups() || inDBRepair;
-        BSONObj order = idx.keyPattern();
+        bool dupsAllowed = !idx->unique() || ignoreUniqueIndex(idx->getOnDisk());
+        bool dropDups = idx->dropDups() || inDBRepair;
+        BSONObj order = idx->keyPattern();
 
-        getDur().writingDiskLoc(idx.head).Null();
+        getDur().writingDiskLoc(idx->getOnDisk().head).Null();
 
         if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) )
             printMemInfo( "before index start" );
@@ -206,17 +209,16 @@ namespace mongo {
         /* get and sort all the keys ----- */
         ProgressMeterHolder pm(op->setMessage("index: (1/3) external sort",
                                               "Index: (1/3) External Sort Progress",
-                                              d->numRecords(),
+                                              collection->numRecords(),
                                               10));
         SortPhaseOne phase1;
-        addKeysToPhaseOne(d, ns, idx, order, &phase1, d->numRecords(), pm.get(),
-                          mayInterrupt, idxNo );
+        addKeysToPhaseOne(collection, idx, order, &phase1, pm.get(), mayInterrupt );
         pm.finished();
 
         BSONObjExternalSorter& sorter = *(phase1.sorter);
 
         if( phase1.multi ) {
-            d->setIndexIsMultikey(ns, idxNo);
+            collection->getIndexCatalog()->markMultikey( idx );
         }
 
         if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) )
@@ -231,7 +233,7 @@ namespace mongo {
         set<DiskLoc> dupsToDrop;
 
         /* build index --- */
-        if( idx.version() == 0 )
+        if( idx->version() == 0 )
             buildBottomUpPhases2And3<V0>(dupsAllowed,
                                          idx,
                                          sorter,
@@ -242,7 +244,7 @@ namespace mongo {
                                          pm,
                                          t,
                                          mayInterrupt);
-        else if( idx.version() == 1 ) 
+        else if( idx->version() == 1 )
             buildBottomUpPhases2And3<V1>(dupsAllowed,
                                          idx,
                                          sorter,
@@ -256,27 +258,28 @@ namespace mongo {
         else
             verify(false);
 
-        if( dropDups ) 
+        if( dropDups )
             log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl;
 
-        BtreeBasedBuilder::doDropDups(ns, d, dupsToDrop, mayInterrupt);
+        doDropDups(collection, dupsToDrop, mayInterrupt);
 
         return phase1.n;
     }
 
-    void BtreeBasedBuilder::doDropDups(const char* ns, NamespaceDetails* d,
+    void BtreeBasedBuilder::doDropDups(Collection* collection,
                                        const set<DiskLoc>& dupsToDrop, bool mayInterrupt) {
-
+        string ns = collection->ns().ns();
         for( set<DiskLoc>::const_iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); ++i ) {
             RARELY killCurrentOp.checkForInterrupt( !mayInterrupt );
-            theDataFileMgr.deleteRecord( d,
-                                         ns,
-                                         i->rec(),
-                                         *i,
-                                         false /* cappedOk */,
-                                         true /* noWarn */,
-                                         isMaster( ns ) /* logOp */ );
+            BSONObj toDelete;
+            collection->deleteDocument( *i,
+                                        false /* cappedOk */,
+                                        true /* noWarn */,
+                                        &toDelete );
             getDur().commitIfNeeded();
+            if ( isMaster( ns.c_str() ) ) {
+                logOp( "d", ns.c_str(), toDelete );
+            }
         }
     }
 
diff --git a/src/mongo/db/index/btree_based_builder.h b/src/mongo/db/index/btree_based_builder.h
index 443f07f76d6..50510a33417 100644
--- a/src/mongo/db/index/btree_based_builder.h
+++ b/src/mongo/db/index/btree_based_builder.h
@@ -42,8 +42,10 @@ namespace IndexUpdateTests {
 
 namespace mongo {
 
+    class Collection;
     class BSONObjExternalSorter;
     class ExternalSortComparison;
+    class IndexDescriptor;
     class IndexDetails;
     class NamespaceDetails;
     class ProgressMeter;
@@ -55,8 +57,8 @@ namespace mongo {
         /**
          * Want to build an index?  Call this.  Throws DBException.
          */
-        static uint64_t fastBuildIndex(const char* ns, NamespaceDetails* d, IndexDetails& idx,
-                                       bool mayInterrupt, int idxNo);
+        static uint64_t fastBuildIndex(Collection* collection, IndexDescriptor* descriptor,
+                                       bool mayInterrupt);
         static DiskLoc makeEmptyIndex(const IndexDetails& idx);
         static ExternalSortComparison* getComparison(int version, const BSONObj& keyPattern);
 
@@ -67,20 +69,18 @@ namespace mongo {
         friend class IndexUpdateTests::InterruptDoDropDups;
 
 
-        static void addKeysToPhaseOne(NamespaceDetails* d, const char* ns, const IndexDetails& idx,
+        static void addKeysToPhaseOne(Collection* collection, IndexDescriptor* idx,
                                       const BSONObj& order, SortPhaseOne* phaseOne,
-                                      int64_t nrecords, ProgressMeter* progressMeter,
-                                      bool mayInterrupt,
-                                      int idxNo);
+                                      ProgressMeter* progressMeter, bool mayInterrupt );
 
-        static void doDropDups(const char* ns, NamespaceDetails* d, const set<DiskLoc>& dupsToDrop,
+        static void doDropDups(Collection* collection, const set<DiskLoc>& dupsToDrop,
                                bool mayInterrupt );
     };
 
     // Exposed for testing purposes.
     template< class V >
     void buildBottomUpPhases2And3( bool dupsAllowed,
-                                   IndexDetails& idx,
+                                   IndexDescriptor* idx,
                                    BSONObjExternalSorter& sorter,
                                    bool dropDups,
                                    set<DiskLoc>& dupsToDrop,
diff --git a/src/mongo/db/index/index_descriptor.h b/src/mongo/db/index/index_descriptor.h
index ddbfcfde403..9546ff8b61d 100644
--- a/src/mongo/db/index/index_descriptor.h
+++ b/src/mongo/db/index/index_descriptor.h
@@ -35,6 +35,8 @@
 #include "mongo/db/namespace_details.h"  // For NamespaceDetails.
 #include "mongo/db/structure/collection.h"
 
+#include "mongo/util/stacktrace.h"
+
 namespace mongo {
 
     class IndexCatalog;
@@ -62,7 +64,8 @@ namespace mongo {
          */
         IndexDescriptor(Collection* collection, int indexNumber, OnDiskIndexData* data,
                         BSONObj infoObj)
-            : _collection(collection), _indexNumber(indexNumber), _onDiskData(data),
+            : _magic(123987),
+              _collection(collection), _indexNumber(indexNumber), _onDiskData(data),
               _infoObj(infoObj.getOwned()),
               _numFields(infoObj.getObjectField("key").nFields()),
               _keyPattern(infoObj.getObjectField("key").getOwned()),
@@ -80,7 +83,10 @@ namespace mongo {
             if ( e.isNumber() ) {
                 _version = e.numberInt();
             }
+        }
 
+        ~IndexDescriptor() {
+            _magic = 555;
         }
 
         // XXX this is terrible
@@ -97,17 +103,17 @@ namespace mongo {
          * Example: {geo: "2dsphere", nonGeo: 1}
          * Example: {foo: 1, bar: -1}
          */
-        const BSONObj& keyPattern() const { return _keyPattern; }
+        const BSONObj& keyPattern() const { _checkOk(); return _keyPattern; }
 
         // How many fields do we index / are in the key pattern?
-        int getNumFields() const { return _numFields; }
+        int getNumFields() const { _checkOk(); return _numFields; }
 
         //
         // Information about the index's namespace / collection.
         //
 
         // Return the name of the index.
-        const string& indexName() const { return _indexName; }
+        const string& indexName() const { _checkOk(); return _indexName; }
 
         // Return the name of the indexed collection.
         const string& parentNS() const { return _parentNS; }
@@ -132,9 +138,9 @@ namespace mongo {
         bool isSparse() const { return _sparse; }
 
         // Is this index multikey?
-        bool isMultikey() const { return _collection->details()->isMultikey(_indexNumber); }
+        bool isMultikey() const { _checkOk(); return _collection->details()->isMultikey(_indexNumber); }
 
-        bool isIdIndex() const { return _isIdIndex; }
+        bool isIdIndex() const { _checkOk(); return _isIdIndex; }
 
         //
         // Properties that are Index-specific.
@@ -149,20 +155,20 @@ namespace mongo {
         //
 
         // Return the memory-mapped index data block.
-        OnDiskIndexData& getOnDisk() { return* _onDiskData; }
+        OnDiskIndexData& getOnDisk() { _checkOk(); return *_onDiskData; }
 
         // Return the mutable head of the index.
-        DiskLoc& getHead() { return _onDiskData->head; }
+        const DiskLoc& getHead() const { _checkOk(); return _onDiskData->head; }
 
         // Return a (rather compact) string representation.
-        string toString() const { return _infoObj.toString(); }
+        string toString() const { _checkOk(); return _infoObj.toString(); }
 
         // Return the info object.
-        const BSONObj& infoObj() const { return _infoObj; }
+        const BSONObj& infoObj() const { _checkOk(); return _infoObj; }
 
         // Set multikey attribute.  We never unset it.
         void setMultikey() {
-            _collection->details()->setIndexIsMultikey(parentNS().c_str(), _indexNumber);
+            _collection->getIndexCatalog()->markMultikey( this );
         }
 
         // Is this index being created in the background?
@@ -178,8 +184,17 @@ namespace mongo {
 
     private:
 
+        void _checkOk() const {
+            if ( _magic == 123987 )
+                return;
+            log() << "uh oh: " << (void*)(this) << " " << _magic;
+            verify(0);
+        }
+
         int getIndexNumber() const { return _indexNumber; }
 
+        int _magic;
+
         // Related catalog information of the parent collection
         Collection* _collection;
 
diff --git a/src/mongo/db/index_legacy.cpp b/src/mongo/db/index_legacy.cpp
index ef98839c29c..3692f0fd7d3 100644
--- a/src/mongo/db/index_legacy.cpp
+++ b/src/mongo/db/index_legacy.cpp
@@ -87,9 +87,9 @@ namespace mongo {
     }
 
     // static
-    void IndexLegacy::postBuildHook(Collection* collection, IndexDescriptor* desc) {
+    void IndexLegacy::postBuildHook(Collection* collection, const BSONObj& keyPattern) {
         // If it's an FTS index, we want to set the power of 2 flag.
-        string pluginName = collection->getIndexCatalog()->getAccessMethodName(desc->keyPattern());
+        string pluginName = collection->getIndexCatalog()->getAccessMethodName(keyPattern);
         if (IndexNames::TEXT == pluginName || IndexNames::TEXT_INTERNAL == pluginName) {
             NamespaceDetails* nsd = collection->details();
             if (nsd->setUserFlag(NamespaceDetails::Flag_UsePowerOf2Sizes)) {
diff --git a/src/mongo/db/index_legacy.h b/src/mongo/db/index_legacy.h
index 14de074c7d4..f9f96e1f4d2 100644
--- a/src/mongo/db/index_legacy.h
+++ b/src/mongo/db/index_legacy.h
@@ -73,7 +73,7 @@ namespace mongo {
          * This is a no-op unless the index is a FTS index.  In that case, we set the flag for using
          * power of 2 sizes for space allocation.
          */
-        static void postBuildHook(Collection* collection, IndexDescriptor* descriptor);
+        static void postBuildHook(Collection* collection, const BSONObj& keyPattern );
     };
 
 }  // namespace mongo
diff --git a/src/mongo/db/namespace_details.cpp b/src/mongo/db/namespace_details.cpp
index 0bff1f5823a..d9428af758c 100644
--- a/src/mongo/db/namespace_details.cpp
+++ b/src/mongo/db/namespace_details.cpp
@@ -436,7 +436,7 @@ namespace mongo {
         return e;
     }
 
-    void NamespaceDetails::setIndexIsMultikey(const char *thisns, int i, bool multikey) {
+    bool NamespaceDetails::setIndexIsMultikey(int i, bool multikey) {
         massert(16577, "index number greater than NIndexesMax", i < NIndexesMax );
 
         unsigned long long mask = 1ULL << i;
@@ -444,7 +444,7 @@ namespace mongo {
         if (multikey) {
             // Shortcut if the bit is already set correctly
             if (_multiKeyIndexBits & mask) {
-                return;
+                return false;
             }
 
             *getDur().writing(&_multiKeyIndexBits) |= mask;
@@ -452,7 +452,7 @@ namespace mongo {
         else {
             // Shortcut if the bit is already set correctly
             if (!(_multiKeyIndexBits & mask)) {
-                return;
+                return false;
             }
 
             // Invert mask: all 1's except a 0 at the ith bit
@@ -460,9 +460,7 @@ namespace mongo {
             *getDur().writing(&_multiKeyIndexBits) &= mask;
         }
 
-        Collection* collection = cc().database()->getCollection( thisns );
-        if ( collection )
-            collection->infoCache()->clearQueryCache();
+        return true;
     }
 
     IndexDetails& NamespaceDetails::getNextIndexDetails(const char* thisns) {
@@ -753,7 +751,7 @@ namespace mongo {
         d->idx( getTotalIndexCount() ) = IndexDetails();
     }
 
-    void NamespaceDetails::swapIndex( const char* ns, int a, int b ) {
+    void NamespaceDetails::swapIndex( int a, int b ) {
 
         // flip main meta data
         IndexDetails temp = idx(a);
@@ -762,8 +760,8 @@ namespace mongo {
 
         // flip multi key bits
         bool tempMultikey = isMultikey(a);
-        setIndexIsMultikey( ns, a, isMultikey(b) );
-        setIndexIsMultikey( ns, b, tempMultikey );
+        setIndexIsMultikey( a, isMultikey(b) );
+        setIndexIsMultikey( b, tempMultikey );
     }
 
     void NamespaceDetails::orphanDeletedList() {
diff --git a/src/mongo/db/namespace_details.h b/src/mongo/db/namespace_details.h
index 5a2552c4686..bc3fd7aa468 100644
--- a/src/mongo/db/namespace_details.h
+++ b/src/mongo/db/namespace_details.h
@@ -283,7 +283,11 @@ namespace mongo {
            for these, we have to do some dedup work on queries.
         */
         bool isMultikey(int i) const { return (_multiKeyIndexBits & (((unsigned long long) 1) << i)) != 0; }
-        void setIndexIsMultikey(const char *thisns, int i, bool multikey = true);
+
+        /**
+         * @return - if any state was changed
+         */
+        bool setIndexIsMultikey(int i, bool multikey = true);
 
         /**
          * This fetches the IndexDetails for the next empty index slot. The caller must populate
@@ -473,7 +477,7 @@ namespace mongo {
          * a and b are 2 index ids, whose contents will be swapped
          * must have a lock on the entire collection to do this
          */
-        void swapIndex( const char* ns, int a, int b );
+        void swapIndex( int a, int b );
 
         DiskLoc _alloc(const StringData& ns, int len);
         void maybeComplain( const StringData& ns, int len ) const;
diff --git a/src/mongo/db/ops/delete.cpp b/src/mongo/db/ops/delete.cpp
index 86e83463d54..8e276e89e75 100644
--- a/src/mongo/db/ops/delete.cpp
+++ b/src/mongo/db/ops/delete.cpp
@@ -127,7 +127,9 @@ namespace mongo {
                 // XXX: do we want to buffer docs and delete them in a group rather than
                 // saving/restoring state repeatedly?
                 runner->saveState();
-                currentClient.get()->database()->getCollection(ns)->deleteDocument(rloc);
+                Collection* collection = currentClient.get()->database()->getCollection(ns);
+                verify( collection );
+                collection->deleteDocument(rloc);
                 runner->restoreState();
 
                 nDeleted++;
diff --git a/src/mongo/db/repl/rs.h b/src/mongo/db/repl/rs.h
index 1d8fce9fc72..16b387a536b 100644
--- a/src/mongo/db/repl/rs.h
+++ b/src/mongo/db/repl/rs.h
@@ -30,10 +30,11 @@
 
 #pragma once
 
+#include "mongo/bson/optime.h"
 #include "mongo/db/commands.h"
+#include "mongo/db/index/index_descriptor.h"
 #include "mongo/db/storage/index_details.h"
 #include "mongo/db/repl/oplogreader.h"
-#include "mongo/bson/optime.h"
 #include "mongo/db/repl/rs_config.h"
 #include "mongo/db/repl/rs_exception.h"
 #include "mongo/db/repl/rs_member.h"
@@ -737,6 +738,34 @@ namespace mongo {
             _hbinfo.health = 1.0;
     }
 
+    inline bool ignoreUniqueIndex(IndexDescriptor* idx) {
+        if (!idx->unique()) {
+            return false;
+        }
+        if (!theReplSet) {
+            return false;
+        }
+        // see SERVER-6671
+        MemberState ms = theReplSet->state();
+        if (! ((ms == MemberState::RS_STARTUP2) ||
+               (ms == MemberState::RS_RECOVERING) ||
+               (ms == MemberState::RS_ROLLBACK))) {
+            return false;
+        }
+        // 2 is the oldest oplog version where operations
+        // are fully idempotent.
+        if (theReplSet->oplogVersion < 2) {
+            return false;
+        }
+        // Never ignore _id index
+        if (idx->isIdIndex()) {
+            return false;
+        }
+
+        return true;
+    }
+
+
     inline bool ignoreUniqueIndex(IndexDetails& idx) {
         if (!idx.unique()) {
             return false;
@@ -760,7 +789,7 @@ namespace mongo {
         if (idx.isIdIndex()) {
             return false;
         }
-        
+
         return true;
     }