auto-reload chunk manager to check version not reset after third staleconfig SERVER-3889

4 files changed, 26 insertions, 15 deletions

diff --git a/s/config.cpp b/s/config.cpp
index 23475ebd9c2..357536267db 100644
--- a/s/config.cpp
+++ b/s/config.cpp
@@ -185,9 +185,9 @@ namespace mongo {
         return true;
     }
 
-    ChunkManagerPtr DBConfig::getChunkManagerIfExists( const string& ns, bool shouldReload ){
+    ChunkManagerPtr DBConfig::getChunkManagerIfExists( const string& ns, bool shouldReload, bool forceReload ){
         try{
-            return getChunkManager( ns, shouldReload );
+            return getChunkManager( ns, shouldReload, forceReload );
         }
         catch( AssertionException& e ){
             warning() << "chunk manager not found for " << ns << causedBy( e ) << endl;
@@ -195,7 +195,7 @@ namespace mongo {
         }
     }
 
-    ChunkManagerPtr DBConfig::getChunkManager( const string& ns , bool shouldReload ) {
+    ChunkManagerPtr DBConfig::getChunkManager( const string& ns , bool shouldReload, bool forceReload ) {
         BSONObj key;
         bool unique;
         ShardChunkVersion oldVersion;
@@ -205,7 +205,7 @@ namespace mongo {
             
             CollectionInfo& ci = _collections[ns];
             
-            bool earlyReload = ! ci.isSharded() && shouldReload;
+            bool earlyReload = ! ci.isSharded() && ( shouldReload || forceReload );
             if ( earlyReload ) {
                 // this is to catch cases where there this is a new sharded collection
                 _reload();
@@ -214,7 +214,7 @@ namespace mongo {
             massert( 10181 ,  (string)"not sharded:" + ns , ci.isSharded() );
             assert( ! ci.key().isEmpty() );
             
-            if ( ! shouldReload || earlyReload )
+            if ( ! ( shouldReload || forceReload ) || earlyReload )
                 return ci.getCM();
 
             key = ci.key().copy();
@@ -225,7 +225,7 @@ namespace mongo {
         
         assert( ! key.isEmpty() );
         
-        if ( oldVersion > 0 ) {
+        if ( oldVersion > 0 && ! forceReload ) {
             ScopedDbConnection conn( configServer.modelServer() , 30.0 );
             BSONObj newest = conn->findOne( ShardNS::chunk , 
                                             Query( BSON( "ns" << ns ) ).sort( "lastmod" , -1 ) );
@@ -240,7 +240,11 @@ namespace mongo {
                     return ci.getCM();
                 }
             }
-            
+
+        }
+        else if( oldVersion == 0 ){
+            warning() << "version 0 found when " << ( forceReload ? "reloading" : "checking" ) << " chunk manager"
+                      << ", collection '" << ns << "' initially detected as sharded" << endl;
         }
 
         // we are not locked now, and want to load a new ChunkManager
@@ -257,8 +261,15 @@ namespace mongo {
         CollectionInfo& ci = _collections[ns];
         massert( 14822 ,  (string)"state changed in the middle: " + ns , ci.isSharded() );
         
-        if ( temp->getVersion() > ci.getCM()->getVersion() ) {
-            // we only want to reset if we're newer
+        bool forced = false;
+        if ( temp->getVersion() > ci.getCM()->getVersion() ||
+            (forced = (temp->getVersion() == ci.getCM()->getVersion() && forceReload ) ) ) {
+
+            if( forced ){
+                warning() << "chunk manager reload forced for collection '" << ns << "', config version is " << temp->getVersion() << endl;
+            }
+
+            // we only want to reset if we're newer or equal and forced
             // otherwise we go into a bad cycle
             ci.resetCM( temp.release() );
         }
diff --git a/s/config.h b/s/config.h
index 90c06cb0223..3b7eb9570ba 100644
--- a/s/config.h
+++ b/s/config.h
@@ -142,8 +142,8 @@ namespace mongo {
          */
         bool isSharded( const string& ns );
 
-        ChunkManagerPtr getChunkManager( const string& ns , bool reload = false );
-        ChunkManagerPtr getChunkManagerIfExists( const string& ns , bool reload = false );
+        ChunkManagerPtr getChunkManager( const string& ns , bool reload = false, bool forceReload = false );
+        ChunkManagerPtr getChunkManagerIfExists( const string& ns , bool reload = false, bool forceReload = false );
 
         /**
          * @return the correct for shard for the ns
diff --git a/s/request.cpp b/s/request.cpp
index 36488cb5617..98740aef29b 100644
--- a/s/request.cpp
+++ b/s/request.cpp
@@ -58,7 +58,7 @@ namespace mongo {
         reset();
     }
 
-    void Request::reset( bool reload ) {
+    void Request::reset( bool reload, bool forceReload ) {
         if ( _m.operation() == dbKillCursors ) {
             return;
         }
@@ -70,7 +70,7 @@ namespace mongo {
         _config = grid.getDBConfig( nsStr );
         if ( reload ) {
             if ( _config->isSharded( nsStr ) )
-                _config->getChunkManager( nsStr , true );
+                _config->getChunkManager( nsStr , true, forceReload );
             else
                 _config->reload();
         }
@@ -137,7 +137,7 @@ namespace mongo {
                 ShardConnection::checkMyConnectionVersions( getns() );
                 if (!staleConfig.justConnection() )
                     sleepsecs( attempt );
-                reset( ! staleConfig.justConnection() );
+                reset( ! staleConfig.justConnection(), attempt >= 2 );
                 _d.markReset();
                 process( attempt + 1 );
                 return;
diff --git a/s/request.h b/s/request.h
index 86a484e378b..d60b95d0f18 100644
--- a/s/request.h
+++ b/s/request.h
@@ -91,7 +91,7 @@ namespace mongo {
 
         void init();
 
-        void reset( bool reload=false );
+        void reset( bool reload=false, bool forceReload = false );
 
     private:
         Message& _m;