SERVER-8751 Bump pdfile minor version on new index plugin creation

Changes:
1) Warn on startup if using old minor version and invalid indexes exist.
2) Treat unknown index plugins created under pre-2.4 as ascending
3) When creating first index using new plugins:
    a) if there are invalid indexes error out
    b) if not pump the pdfile minor version
4) ScanAndOrder no longer supports index plugins (it never should have)

Note:
All text and 2dsphere indexes created prior to this commit will need to be rebuilt.

8 files changed, 156 insertions, 25 deletions

diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 5c934551414..12037d0b6a8 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -293,7 +293,7 @@ namespace mongo {
 
         if ( h->version == 4 && h->versionMinor == 4 ) {
             verify( PDFILE_VERSION == 4 );
-            verify( PDFILE_VERSION_MINOR == 5 );
+            verify( PDFILE_VERSION_MINOR_22_AND_OLDER == 5 );
 
             list<string> colls = db.getCollectionNames( dbName );
             for ( list<string>::iterator i=colls.begin(); i!=colls.end(); i++) {
@@ -341,8 +341,11 @@ namespace mongo {
 
                 log() << "****" << endl;
                 log() << "****" << endl;
-                log() << "need to upgrade database " << dbName << " with pdfile version " << h->version << "." << h->versionMinor << ", "
-                      << "new version: " << PDFILE_VERSION << "." << PDFILE_VERSION_MINOR << endl;
+                log() << "need to upgrade database " << dbName << " "
+                      << "with pdfile version " << h->version << "." << h->versionMinor << ", "
+                      << "new version: "
+                      << PDFILE_VERSION << "." << PDFILE_VERSION_MINOR_22_AND_OLDER
+                      << endl;
                 if ( shouldRepairDatabases ) {
                     // QUESTION: Repair even if file format is higher version than code?
                     log() << "\t starting upgrade" << endl;
@@ -359,6 +362,23 @@ namespace mongo {
                 }
             }
             else {
+                if (h->versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER) {
+                    const string systemIndexes = cc().database()->name + ".system.indexes";
+                    shared_ptr<Cursor> cursor(theDataFileMgr.findAll(systemIndexes));
+                    for ( ; cursor && cursor->ok(); cursor->advance()) {
+                        const BSONObj index = cursor->current();
+                        const BSONObj key = index.getObjectField("key");
+                        const string plugin = IndexPlugin::findPluginName(key);
+                        if (IndexPlugin::existedBefore24(plugin))
+                            continue;
+
+                        log() << "Index " << index << " claims to be of type '" << plugin << "', "
+                              << "which is either invalid or did not exist before v2.4. "
+                              << "See the upgrade section: "
+                              << "http://dochub.mongodb.org/core/upgrade-2.4"
+                              << startupWarningsLog;
+                    }
+                }
                 Database::closeDatabase( dbName.c_str(), dbpath );
             }
         }
diff --git a/src/mongo/db/index.cpp b/src/mongo/db/index.cpp
index 5077faa1894..377679806a4 100644
--- a/src/mongo/db/index.cpp
+++ b/src/mongo/db/index.cpp
@@ -287,6 +287,43 @@ namespace mongo {
         return true;
     }
 
+    static bool needToUpgradeMinorVersion(const string& newPluginName) {
+        if (IndexPlugin::existedBefore24(newPluginName))
+            return false;
+
+        DataFileHeader* dfh = cc().database()->getFile(0)->getHeader();
+        if (dfh->versionMinor == PDFILE_VERSION_MINOR_24_AND_NEWER)
+            return false; // these checks have already been done
+
+        fassert(16737, dfh->versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER);
+
+        return true;
+    }
+
+    static void upgradeMinorVersionOrAssert(const string& newPluginName) {
+        const string systemIndexes = cc().database()->name + ".system.indexes";
+        shared_ptr<Cursor> cursor(theDataFileMgr.findAll(systemIndexes));
+        for ( ; cursor && cursor->ok(); cursor->advance()) {
+            const BSONObj index = cursor->current();
+            const BSONObj key = index.getObjectField("key");
+            const string plugin = IndexPlugin::findPluginName(key);
+            if (IndexPlugin::existedBefore24(plugin))
+                continue;
+
+            const string errmsg = str::stream()
+                << "Found pre-existing index " << index << " with invalid type '" << plugin << "'. "
+                << "Disallowing creation of new index type '" << newPluginName << "'. See "
+                << "http://dochub.mongodb.org/core/index-type-changes"
+                ;
+
+            error() << errmsg << endl;
+            uasserted(16738, errmsg);
+        }
+
+        DataFileHeader* dfh = cc().database()->getFile(0)->getHeader();
+        getDur().writingInt(dfh->versionMinor) = PDFILE_VERSION_MINOR_24_AND_NEWER;
+    }
+
     bool prepareToBuildIndex(const BSONObj& io,
                              bool mayInterrupt,
                              bool god,
@@ -373,6 +410,9 @@ namespace mongo {
             uassert(16734, str::stream() << "Unknown index plugin '" << pluginName << "' "
                                          << "in index "<< key
                    , plugin);
+
+            if (needToUpgradeMinorVersion(pluginName))
+                upgradeMinorVersionOrAssert(pluginName);
         }
 
         { 
@@ -414,19 +454,25 @@ namespace mongo {
         return true;
     }
 
-    void IndexSpec::reset( const IndexDetails * details ) {
+    void IndexSpec::reset(const IndexDetails * details) {
+        const DataFileHeader* dfh = cc().database()->getFile(0)->getHeader();
+        IndexSpec::PluginRules rules = dfh->versionMinor == PDFILE_VERSION_MINOR_24_AND_NEWER
+                                            ? IndexSpec::RulesFor24
+                                            : IndexSpec::RulesFor22
+                                            ;
+
         _details = details;
-        reset( details->info );
+        reset(details->info, rules);
     }
 
-    void IndexSpec::reset( const BSONObj& _info ) {
+    void IndexSpec::reset(const BSONObj& _info, PluginRules rules) {
         info = _info;
         keyPattern = info["key"].embeddedObjectUserCheck();
         if ( keyPattern.objsize() == 0 ) {
             out() << info.toString() << endl;
             verify(false);
         }
-        _init();
+        _init(rules);
     }
 
     void IndexChanges::dupCheck(IndexDetails& idx, DiskLoc curObjLoc) {
diff --git a/src/mongo/db/indexkey.cpp b/src/mongo/db/indexkey.cpp
index 91675c32602..616dbd9a499 100644
--- a/src/mongo/db/indexkey.cpp
+++ b/src/mongo/db/indexkey.cpp
@@ -23,6 +23,9 @@
 #include "../util/stringutils.h"
 #include "mongo/util/mongoutils/str.h"
 #include "../util/text.h"
+#include "mongo/db/client.h"
+#include "mongo/db/database.h"
+#include "mongo/db/pdfile.h"
 #include "mongo/db/queryutil.h"
 
 namespace mongo {
@@ -78,7 +81,7 @@ namespace mongo {
         return l.woCompare( r , _spec->keyPattern );
     }
 
-    void IndexSpec::_init() {
+    void IndexSpec::_init(PluginRules rules) {
         verify( keyPattern.objsize() );
 
         // some basics
@@ -123,12 +126,41 @@ namespace mongo {
             string pluginName = IndexPlugin::findPluginName( keyPattern );
             if ( pluginName.size() ) {
                 IndexPlugin * plugin = IndexPlugin::get( pluginName );
-                if ( ! plugin ) {
-                    log() << "warning: can't find plugin [" << pluginName << "]" << endl;
+
+                switch (rules) {
+                case NoPlugins:
+                    uasserted(16735,
+                              str::stream()
+                                << "Attempting to use index type '" << pluginName << "' "
+                                << "where index types are not allowed (1 or -1 only).");
+                    break;
+
+                case RulesFor22: {
+                    if ( ! plugin ) {
+                        log() << "warning: can't find plugin [" << pluginName << "]" << endl;
+                    }
+
+                    if (!IndexPlugin::existedBefore24(pluginName)) {
+                        warning() << "Treating index " << info << " as ascending since "
+                                  << "it was created before 2.4 and '" << pluginName << "' "
+                                  << "was not a valid type at that time."
+                                  << endl;
+
+                        plugin = NULL;
+                    }
+                    break;
                 }
-                else {
-                    _indexType.reset( plugin->generate( this ) );
+                case RulesFor24:
+                    // This assert will be triggered when downgrading from a future version that
+                    // supports an index plugin unsupported by this version.
+                    uassert(16736, str::stream() << "Invalid index type '" << pluginName << "' "
+                                                 << "in index " << info
+                           , plugin);
+                    break;
                 }
+
+                if (plugin)
+                    _indexType.reset( plugin->generate( this ) );
             }
         }
 
diff --git a/src/mongo/db/indexkey.h b/src/mongo/db/indexkey.h
index ff7975fca40..187f2fe48bc 100644
--- a/src/mongo/db/indexkey.h
+++ b/src/mongo/db/indexkey.h
@@ -142,6 +142,19 @@ namespace mongo {
          */
         static string findPluginName( const BSONObj& keyPattern );
 
+        /**
+         * True if is a regular (non-plugin) index or uses a plugin that existed before 2.4.
+         * These plugins are grandfathered in and allowed to exist in DBs with
+         * PDFILE_MINOR_VERSION_22_AND_OLDER
+         */
+        static bool existedBefore24(const string& name) {
+            return name.empty()
+                || name == "2d"
+                || name == "geoHaystack"
+                || name == "hashed"
+                ;
+        }
+
     private:
         string _name;
         static map<string,IndexPlugin*> * _plugins;
@@ -152,6 +165,12 @@ namespace mongo {
        */
     class IndexSpec {
     public:
+        enum PluginRules {
+            NoPlugins,
+            RulesFor22, // if !IndexPlugin::existedBefore24() treat as ascending
+            RulesFor24, // allow new plugins but error if unknown
+        };
+
         BSONObj keyPattern; // e.g., { name : 1 }
         BSONObj info; // this is the same as IndexDetails::info.obj()
 
@@ -159,22 +178,25 @@ namespace mongo {
             : _details(0) , _finishedInit(false) {
         }
 
-        explicit IndexSpec( const BSONObj& k , const BSONObj& m = BSONObj() )
+        explicit IndexSpec(const BSONObj& k, const BSONObj& m=BSONObj(),
+                           PluginRules rules=RulesFor24)
             : keyPattern(k) , info(m) , _details(0) , _finishedInit(false) {
-            _init();
+            _init(rules);
         }
 
         /**
            this is a DiscLoc of an IndexDetails info
            should have a key field
          */
-        explicit IndexSpec( const DiskLoc& loc ) {
-            reset( loc );
+        explicit IndexSpec(const DiskLoc& loc, PluginRules rules=RulesFor24) {
+            reset(loc, rules);
         }
 
-        void reset( const BSONObj& info );
-        void reset( const DiskLoc& infoLoc ) { reset(infoLoc.obj()); }
-        void reset( const IndexDetails * details );
+        void reset(const BSONObj& info, PluginRules rules=RulesFor24);
+        void reset(const IndexDetails * details); // determines rules based on pdfile version
+        void reset(const DiskLoc& infoLoc, PluginRules rules=RulesFor24) {
+            reset(infoLoc.obj(), rules);
+        }
 
         void getKeys( const BSONObj &obj, BSONObjSet &keys ) const;
 
@@ -232,7 +254,7 @@ namespace mongo {
         shared_ptr<IndexType> _indexType;
         const IndexDetails * _details;
 
-        void _init();
+        void _init(PluginRules rules);
 
         friend class IndexType;
         friend class KeyGeneratorV0;
diff --git a/src/mongo/db/pdfile.h b/src/mongo/db/pdfile.h
index c22ed679edc..094c2e28246 100644
--- a/src/mongo/db/pdfile.h
+++ b/src/mongo/db/pdfile.h
@@ -460,7 +460,11 @@ namespace mongo {
 
         enum { HeaderSize = 8192 };
 
-        bool isCurrentVersion() const { return ( version == PDFILE_VERSION ) && ( versionMinor == PDFILE_VERSION_MINOR ); }
+        bool isCurrentVersion() const {
+            return version == PDFILE_VERSION && ( versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER
+                                               || versionMinor == PDFILE_VERSION_MINOR_24_AND_NEWER
+                                                );
+        }
 
         bool uninitialized() const { return version == 0; }
 
@@ -491,7 +495,7 @@ namespace mongo {
                 DataFileHeader *h = getDur().writing(this);
                 h->fileLength = filelength;
                 h->version = PDFILE_VERSION;
-                h->versionMinor = PDFILE_VERSION_MINOR;
+                h->versionMinor = PDFILE_VERSION_MINOR_22_AND_OLDER; // All dbs start like this
                 h->unused.set( fileno, HeaderSize );
                 verify( (data-(char*)this) == HeaderSize );
                 h->unusedLength = fileLength - HeaderSize - 16;
diff --git a/src/mongo/db/pdfile_version.h b/src/mongo/db/pdfile_version.h
index a6b33097533..0c64c7935c8 100644
--- a/src/mongo/db/pdfile_version.h
+++ b/src/mongo/db/pdfile_version.h
@@ -20,6 +20,12 @@ namespace mongo {
 
     // pdfile versions
     const int PDFILE_VERSION = 4;
-    const int PDFILE_VERSION_MINOR = 5;
+    const int PDFILE_VERSION_MINOR_22_AND_OLDER = 5;
+    const int PDFILE_VERSION_MINOR_24_AND_NEWER = 6;
+
+    // For backward compatibility with versions before 2.4.0 all new DBs start
+    // with PDFILE_VERSION_MINOR_22_AND_OLDER and are converted when the first
+    // index using a new plugin is created. See the logic in
+    // prepareToBuildIndex() and upgradeMinorVersionOrAssert() for details
 
 } // namespace mongo
diff --git a/src/mongo/db/scanandorder.h b/src/mongo/db/scanandorder.h
index f104cfe51fc..4cdb0c77510 100644
--- a/src/mongo/db/scanandorder.h
+++ b/src/mongo/db/scanandorder.h
@@ -34,8 +34,9 @@ namespace mongo {
         FieldRangeVector _keyCutter;
     public:
         KeyType(const BSONObj &pattern, const FieldRangeSet &frs):
-        _spec((verify(!pattern.isEmpty()),pattern)),
+        _spec(pattern, BSONObj(), IndexSpec::NoPlugins),
         _keyCutter(frs, _spec, 1) {
+            verify(!pattern.isEmpty());
         }
 
         /**
diff --git a/src/mongo/util/version.cpp b/src/mongo/util/version.cpp
index 6d5d94b98bb..1ee82e86e82 100644
--- a/src/mongo/util/version.cpp
+++ b/src/mongo/util/version.cpp
@@ -109,7 +109,7 @@ namespace mongo {
 
     string mongodVersion() {
         stringstream ss;
-        ss << "db version v" << versionString << ", pdfile version " << PDFILE_VERSION << "." << PDFILE_VERSION_MINOR;
+        ss << "db version v" << versionString;
         return ss.str();
     }