check all shard connections at once so we don't get cascading failure SERVER-1560

7 files changed, 47 insertions, 23 deletions

diff --git a/client/connpool.h b/client/connpool.h
index 94c56bd5854..00570c52d5b 100644
--- a/client/connpool.h
+++ b/client/connpool.h
@@ -96,10 +96,18 @@ namespace mongo {
     
     extern DBConnectionPool pool;
 
+    class AScopedConnection : boost::noncopyable {
+    public:
+        virtual ~AScopedConnection(){}
+        virtual DBClientBase* get() = 0;
+        virtual void done() = 0;
+        virtual string getHost() const = 0;
+    };
+
     /** Use to get a connection from the pool.  On exceptions things
        clean up nicely.
     */
-    class ScopedDbConnection : boost::noncopyable {
+    class ScopedDbConnection : public AScopedConnection {
         const string _host;
         DBClientBase *_conn;
     public:
diff --git a/client/dbclientcursor.cpp b/client/dbclientcursor.cpp
index 863424c77e0..07771bbf939 100644
--- a/client/dbclientcursor.cpp
+++ b/client/dbclientcursor.cpp
@@ -191,15 +191,7 @@ namespace mongo {
         }
     }
 
-    void DBClientCursor::attach( ScopedDbConnection * conn ){
-        assert( _scopedHost.size() == 0 );
-        assert( connector == conn->get() );
-        _scopedHost = conn->getHost();
-        conn->done();
-        connector = 0;
-    }
-
-    void DBClientCursor::attach( ShardConnection * conn ){
+    void DBClientCursor::attach( AScopedConnection * conn ){
         assert( _scopedHost.size() == 0 );
         assert( connector == conn->get() );
         _scopedHost = conn->getHost();
diff --git a/client/dbclientcursor.h b/client/dbclientcursor.h
index 720c0adcff1..51cdc13caed 100644
--- a/client/dbclientcursor.h
+++ b/client/dbclientcursor.h
@@ -25,7 +25,7 @@
 
 namespace mongo {
     
-    class ShardConnection;
+    class AScopedConnection;
     
 	/** Queries return a cursor object */
     class DBClientCursor : boost::noncopyable {
@@ -149,8 +149,7 @@ namespace mongo {
         */
         void decouple() { _ownCursor = false; }
         
-        void attach( ScopedDbConnection * conn );
-        void attach( ShardConnection * conn );
+        void attach( AScopedConnection * conn );
         
     private:
         friend class DBClientBase;
diff --git a/s/request.cpp b/s/request.cpp
index e65aa1532f0..e09c3bc4078 100644
--- a/s/request.cpp
+++ b/s/request.cpp
@@ -122,7 +122,7 @@ namespace mongo {
             catch ( StaleConfigException& staleConfig ){
                 log() << staleConfig.what() << " attempt: " << attempt << endl;
                 uassert( 10195 ,  "too many attempts to update config, failing" , attempt < 5 );
-                
+                ShardConnection::checkMyConnectionVersions( getns() );
                 sleepsecs( attempt );
                 reset( ! staleConfig.justConnection() );
                 _d.markReset();
diff --git a/s/shard.cpp b/s/shard.cpp
index 4ef68c0103c..8ef21a0f798 100644
--- a/s/shard.cpp
+++ b/s/shard.cpp
@@ -98,7 +98,6 @@ namespace mongo {
             reload();
 
             scoped_lock lk( _mutex );
-            
             map<string,Shard>::iterator i = _lookup.find( ident );
             uassert( 13129 , (string)"can't find shard for: " + ident , i != _lookup.end() );
             return i->second;        
@@ -164,6 +163,17 @@ namespace mongo {
         staticShardInfo.getAllShards( all );
     }
 
+    bool Shard::isAShard( const string& ident ){
+        return staticShardInfo.isMember( ident );
+    }
+
+    void Shard::printShardInfo( ostream& out ){
+        vector<Shard> all;
+        getAllShards( all );
+        for ( unsigned i=0; i<all.size(); i++ )
+            out << all[i].toString() << "\n";
+        out.flush();
+    }
     
     BSONObj Shard::runCommand( const string& db , const BSONObj& cmd ) const {
         ScopedDbConnection conn( this );
diff --git a/s/shard.h b/s/shard.h
index a2032b55413..ee807d81033 100644
--- a/s/shard.h
+++ b/s/shard.h
@@ -53,6 +53,8 @@ namespace mongo {
             s.reset( ident );
             return s;
         }
+
+        static bool isAShard( const string& ident );
         
         /**
          * @param ident either name or address
@@ -126,7 +128,8 @@ namespace mongo {
         ShardStatus getStatus() const ;
         
         static void getAllShards( vector<Shard>& all );
-        
+        static void printShardInfo( ostream& out );
+
         /**
          * picks a Shard for more load
          */
@@ -181,7 +184,7 @@ namespace mongo {
         double _writeLock;
     };
 
-    class ShardConnection : boost::noncopyable{
+    class ShardConnection : public AScopedConnection {
     public:
         ShardConnection( const Shard * s , const string& ns );
         ShardConnection( const Shard& s , const string& ns );
@@ -232,7 +235,7 @@ namespace mongo {
         bool runCommand( const string& db , const BSONObj& cmd , BSONObj& res );
 
         /** checks all of my thread local connections for the version of this ns */
-        static void checkMyConnectionVersions( const string &  ns );
+        static void checkMyConnectionVersions( const string & ns );
         
     private:
         void _init();
diff --git a/s/shardconnection.cpp b/s/shardconnection.cpp
index 74a0c5605bd..6c4bb1fe02a 100644
--- a/s/shardconnection.cpp
+++ b/s/shardconnection.cpp
@@ -76,7 +76,7 @@ namespace mongo {
             if ( ! s )
                 s = new Status();
             
-            debug() << "WANT ONE pool avail: " << s->avail << endl;
+            debug( s , addr ) << "WANT ONE pool avail: " << s->avail << endl;
             
             if ( s->avail ){
                 DBClientBase* c = s->avail;
@@ -96,7 +96,8 @@ namespace mongo {
             Status* s = _hosts[addr];
             assert( s );
             if ( s->avail ){
-                delete conn;
+                debug( s , addr ) << "DONE WITH TEMP" << endl;
+                release( addr , conn );
                 return;
             }
             s->avail = conn;
@@ -120,12 +121,23 @@ namespace mongo {
         }
 
         void checkVersions( const string& ns ){
+            vector<Shard> all;
+            Shard::getAllShards( all );
             scoped_lock lk( _mutex );
+            for ( unsigned i=0; i<all.size(); i++ ){
+                Status* &s = _hosts[all[i].getConnString()];
+                if ( ! s )
+                    s = new Status();
+            }
+
             for ( map<string,Status*>::iterator i=_hosts.begin(); i!=_hosts.end(); ++i ){
+                if ( ! Shard::isAShard( i->first ) )
+                    continue;
                 Status* ss = i->second;
                 assert( ss );
-                if ( ss->avail )
-                    checkShardVersion( *ss->avail , ns );
+                if ( ! ss->avail )
+                    ss->avail = pool.get( i->first );
+                checkShardVersion( *ss->avail , ns );
             }
         }
 
@@ -184,7 +196,7 @@ namespace mongo {
         if ( _finishedInit )
             return;
         _finishedInit = true;
-
+        
         if ( _ns.size() ){
             _setVersion = checkShardVersion( *_conn , _ns );
         }