Clean up ReplicaSetMonitor when the whole set has been down for a long time. SERVER-4581

10 files changed, 350 insertions, 56 deletions

diff --git a/jstests/sharding/remove2.js b/jstests/sharding/remove2.js
new file mode 100644
index 00000000000..76a94d6853c
--- /dev/null
+++ b/jstests/sharding/remove2.js
@@ -0,0 +1,178 @@
+// Test that removing and re-adding shard works correctly.
+
+seedString = function(replTest) {
+    members = replTest.getReplSetConfig().members.map(function(elem) { return elem.host; });
+    return replTest.name + '/' + members.join(',');
+};
+
+removeShard = function(st, replTest) {
+    print( "Removing shard with name: " + replTest.name );
+    res = st.admin.runCommand( { removeshard: replTest.name } )
+    printjson(res);
+    assert( res.ok , "failed to start draining shard" );
+    checkRemoveShard = function() {
+        res = st.admin.runCommand( { removeshard: replTest.name } );
+        printjson(res);
+        return res.ok && res.msg == 'removeshard completed successfully';
+    }
+    assert.soon( checkRemoveShard , "failed to remove shard" );
+    replTest.getPrimary().getDB( coll.getDB().getName() ).dropDatabase();
+    print( "Shard removed successfully" );
+};
+
+addShard = function(st, replTest) {
+    seed = seedString(replTest);
+    print( "Adding shard with seed: " + seed );
+    try {
+        assert.eq(true, st.adminCommand({ addshard : seed }));
+    } catch (e) {
+        print("First attempt to addShard failed, trying again")
+        // transport error on first attempt is expected.  Make sure second attempt goes through
+        assert.eq(true, st.adminCommand({ addshard : seed }));
+    }
+    ReplSetTest.awaitRSClientHosts( new Mongo( st.s.host ),
+                                    replTest.getSecondaries(),
+                                    {ok : true, secondary : true} );
+
+    assert.soon( function() {
+        var x = st.chunkDiff( coll.getName() , coll.getDB().getName() );
+        print( "chunk diff: " + x );
+        return x < 2;
+    } , "no balance happened", 60000 );
+
+    try {
+        assert.eq( 300, coll.find().itcount() );
+    } catch (e) {
+        // Expected.  First query might get transport error and need to reconnect.
+        printjson(e);
+        assert.eq( 300, coll.find().itcount() );
+    }
+    print( "Shard added successfully" );
+};
+
+var st = new ShardingTest( testName = "remove2",
+                           numShards = 2,
+                           verboseLevel = 0,
+                           numMongos = 1,
+                           { chunkSize : 1,
+                             rs : true,
+                             rs0 : { nodes : 2 },
+                             rs1 : { nodes : 2 }
+                           });
+
+var rst0 = st._rs[0].test;
+var rst1 = st._rs[1].test;
+
+var conn = new Mongo( st.s.host );
+var coll = conn.getCollection( "test.remove2" );
+coll.drop();
+
+// Decrease how long it will take for rst0 to time out its ReplicaSetMonitor for rst1 when rs1 is shut down
+for( var i = 0; i < rst0.nodes.length; i++ ) {
+    node = rst0.nodes[i];
+    res = node.getDB('admin').runCommand({ setParameter : 1, replMonitorMaxFailedChecks : 1 });
+    printjson( res );
+    assert( res.ok );
+}
+
+st.admin.runCommand({ enableSharding : coll.getDB().getName() });
+st.admin.runCommand({ shardCollection : coll.getFullName(), key: { i : 1 }});
+
+// Setup initial data
+var str = 'a';
+while( str.length < 1024 * 16 ) {
+    str += str;
+}
+for( var i = 0; i < 300; i++ ){
+    coll.insert( { i : i % 10, str : str } );
+}
+
+assert.soon( function() {
+    var x = st.chunkDiff( 'remove2' , "test" ); print( "chunk diff: " + x ); return x < 2;
+} , "no balance happened" );
+
+printjson(res);
+assert(res.ok);
+
+assert.eq( 300, coll.find().itcount() );
+
+st.admin.printShardingStatus();
+
+// Remove shard and add it back in, without shutting it down.
+jsTestLog( "Attempting to remove shard and add it back in" )
+removeShard( st, rst1 );
+addShard(st, rst1 );
+
+
+// Remove shard, restart set, then add it back in.
+jsTestLog( "Attempting to remove shard, restart the set, and then add it back in" )
+originalSeed = seedString(rst1);
+
+removeShard( st, rst1 );
+rst1.stopSet();
+print( "Sleeping for 20 seconds to let the other shard's ReplicaSetMonitor time out" );
+sleep( 20000 ); // 1 failed check should take 10 seconds, sleep for 20 just to be safe
+
+rst1.startSet();
+rst1.initiate();
+rst1.awaitReplication();
+
+assert.eq( originalSeed, seedString(rst1), "Set didn't come back up with the same hosts as before" );
+addShard( st, rst1 );
+
+
+// Shut down shard and wait for its ReplicaSetMonitor to be cleaned up, then start it back up and use it.
+// TODO: test this both with AND without waiting for the ReplicaSetMonitor to be cleaned up.
+// This part doesn't pass, even without cleaning up the ReplicaSetMonitor - see SERVER-5900.
+/*printjson( conn.getDB('admin').runCommand({movePrimary : 'test2', to : rst1.name}) );
+printjson( conn.getDB('admin').runCommand({setParameter : 1, replMonitorMaxFailedChecks : 5}) );
+jsTestLog( "Shutting down set" )
+rst1.stopSet();
+jsTestLog( "sleeping for 20 seconds to make sure ReplicaSetMonitor gets cleaned up");
+sleep(20000); // 1 failed check should take 10 seconds, sleep for 20 just to be safe
+
+// Should fail since rst1 is the primary for test2
+assert.throws(function() {conn.getDB('test2').foo.find().itcount()});
+jsTestLog( "Bringing set back up" );
+rst1.startSet();
+rst1.initiate();
+rst1.awaitReplication();
+
+jsTestLog( "Checking that set is usable again" );
+//conn.getDB('admin').runCommand({flushRouterConfig:1}); // Uncommenting this makes test pass
+conn.getDB('test2').foo.insert({a:1});
+gle = conn.getDB('test2').runCommand('getLastError');
+if ( !gle.ok ) {
+    // Expected.  First write will fail and need to re-connect
+    print( "write failed" );
+    printjson( gle );
+    conn.getDB('test2').foo.insert({a:1});
+    assert( conn.getDB('test2').getLastErrorObj().ok );
+}
+
+assert.eq( 1, conn.getDB('test2').foo.find().itcount() );
+assert( conn.getDB('test2').dropDatabase().ok );*/
+
+
+// Remove shard and add a new shard with the same replica set and shard name, but different ports.
+jsTestLog( "Attempt removing shard and adding a new shard with the same Replica Set name" );
+removeShard( st, rst1 );
+rst1.stopSet();
+print( "Sleeping for 20 seconds to let the other shard's ReplicaSetMonitor time out" );
+sleep( 20000 );
+
+
+var rst2 = new ReplSetTest({name : rst1.name, nodes : 2, startPort : rst1.startPort + 1500, useHostName : true});
+rst2.startSet();
+rst2.initiate();
+rst2.awaitReplication();
+
+addShard( st, rst2 );
+printjson( st.admin.runCommand({movePrimary : 'test2', to : rst2.name}) );
+
+assert.eq( 300, coll.find().itcount() );
+conn.getDB('test2').foo.insert({a:1});
+assert.eq( 1, conn.getDB('test2').foo.find().itcount() );
+
+jsTestLog( "finishing!" )
+st.stop()
diff --git a/src/mongo/client/connpool.cpp b/src/mongo/client/connpool.cpp
index 0011f8e2595..6e5c7f51f36 100644
--- a/src/mongo/client/connpool.cpp
+++ b/src/mongo/client/connpool.cpp
@@ -22,12 +22,17 @@
 #include "connpool.h"
 #include "syncclusterconnection.h"
 #include "../s/shard.h"
+#include "mongo/client/dbclient_rs.h"
 
 namespace mongo {
 
     // ------ PoolForHost ------
 
     PoolForHost::~PoolForHost() {
+        clear();
+    }
+
+    void PoolForHost::clear() {
         while ( ! _pool.empty() ) {
             StoredConnection sc = _pool.top();
             delete sc.conn;
diff --git a/src/mongo/client/connpool.h b/src/mongo/client/connpool.h
index 4467e3e9212..840b49c7d28 100644
--- a/src/mongo/client/connpool.h
+++ b/src/mongo/client/connpool.h
@@ -56,6 +56,9 @@ namespace mongo {
          */
         DBClientBase * get( DBConnectionPool * pool , double socketTimeout );
 
+        // Deletes all connections in the pool
+        void clear();
+
         void done( DBConnectionPool * pool , DBClientBase * c );
 
         void flush();
diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp
index bac76263cd8..4b3380aca2b 100644
--- a/src/mongo/client/dbclient_rs.cpp
+++ b/src/mongo/client/dbclient_rs.cpp
@@ -88,11 +88,14 @@ namespace mongo {
         return seedStr;
     }
 
+    // Must already be in _setsLock when constructing a new ReplicaSetMonitor. This is why you
+    // should only create ReplicaSetMonitors from ReplicaSetMonitor::get and
+    // ReplicaSetMonitor::createIfNeeded.
     ReplicaSetMonitor::ReplicaSetMonitor( const string& name , const vector<HostAndPort>& servers )
         : _lock( "ReplicaSetMonitor instance" ),
           _checkConnectionLock( "ReplicaSetMonitor check connection lock" ),
           _name( name ), _master(-1),
-          _nextSlave(0), _localThresholdMillis(cmdLine.defaultLocalThresholdMillis) {
+          _nextSlave(0), _failedChecks(0), _localThresholdMillis(cmdLine.defaultLocalThresholdMillis) {
 
         uassert( 13642 , "need at least 1 node for a replica set" , servers.size() > 0 );
 
@@ -128,35 +131,60 @@ namespace mongo {
         // Check everything to get the first data
         _check( true );
 
+        _setServers.insert( pair<string, vector<HostAndPort> >(name, servers) );
+
         log() << "replica set monitor for replica set " << _name << " started, address is " << getServerAddress() << endl;
 
     }
 
+    // Must already be in _setsLock when destroying a ReplicaSetMonitor. This is why you should only
+    // delete ReplicaSetMonitors from ReplicaSetMonitor::remove.
     ReplicaSetMonitor::~ReplicaSetMonitor() {
+        scoped_lock lk ( _lock );
+        log() << "deleting replica set monitor for: " << _getServerAddress_inlock() << endl;
+        _cacheServerAddresses_inlock();
         _nodes.clear();
         _master = -1;
     }
 
-    ReplicaSetMonitorPtr ReplicaSetMonitor::get( const string& name , const vector<HostAndPort>& servers ) {
+    void ReplicaSetMonitor::_cacheServerAddresses_inlock() {
+        // Save list of current set members so that the monitor can be rebuilt if needed.
+        vector<HostAndPort>& servers = _setServers[_name];
+        servers.clear();
+        for ( vector<Node>::iterator it = _nodes.begin(); it < _nodes.end(); ++it ) {
+            servers.push_back( it->addr );
+        }
+    }
+
+    void ReplicaSetMonitor::createIfNeeded( const string& name , const vector<HostAndPort>& servers ) {
         scoped_lock lk( _setsLock );
         ReplicaSetMonitorPtr& m = _sets[name];
         if ( ! m )
             m.reset( new ReplicaSetMonitor( name , servers ) );
 
         replicaSetMonitorWatcher.safeGo();
-
-        return m;
     }
 
-    ReplicaSetMonitorPtr ReplicaSetMonitor::get( const string& name ) {
+    ReplicaSetMonitorPtr ReplicaSetMonitor::get( const string& name , const bool createFromSeed ) {
         scoped_lock lk( _setsLock );
         map<string,ReplicaSetMonitorPtr>::const_iterator i = _sets.find( name );
-        if ( i == _sets.end() ) 
-            return ReplicaSetMonitorPtr();
-        return i->second;
+        if ( i != _sets.end() ) {
+            return i->second;
+        }
+        if ( createFromSeed ) {
+            map<string,vector<HostAndPort> >::const_iterator j = _setServers.find( name );
+            if ( j != _setServers.end() ) {
+                log(4) << "Creating ReplicaSetMonitor from cached address" << endl;
+                ReplicaSetMonitorPtr& m = _sets[name];
+                verify( !m );
+                m.reset( new ReplicaSetMonitor( name, j->second ) );
+                replicaSetMonitorWatcher.safeGo();
+                return m;
+            }
+        }
+        return ReplicaSetMonitorPtr();
     }
 
-
     void ReplicaSetMonitor::checkAll( bool checkAllSecondaries ) {
         set<string> seen;
 
@@ -179,14 +207,30 @@ namespace mongo {
                 break;
 
             m->check( checkAllSecondaries );
+            {
+                scoped_lock lk( _setsLock );
+                if ( m->_failedChecks >= _maxFailedChecks ) {
+                    log() << "Replica set " << m->getName() << " was down for " << m->_failedChecks
+                          << " checks in a row. Stopping polled monitoring of the set." << endl;
+                    _remove_inlock( m->getName() );
+                }
+            }
         }
 
 
     }
 
-    void ReplicaSetMonitor::remove( const string& name ) {
+    void ReplicaSetMonitor::remove( const string& name, bool clearSeedCache ) {
         scoped_lock lk( _setsLock );
+        _remove_inlock( name, clearSeedCache );
+    }
+
+    void ReplicaSetMonitor::_remove_inlock( const string& name, bool clearSeedCache ) {
+        log(2) << "Removing ReplicaSetMonitor for " << name << " from replica set table" << endl;
         _sets.erase( name );
+        if ( clearSeedCache ) {
+            _setServers.erase( name );
+        }
     }
 
     void ReplicaSetMonitor::setConfigChangeHook( ConfigChangeHook hook ) {
@@ -617,7 +661,7 @@ namespace mongo {
 
         int newMaster = -1;
         shared_ptr<DBClientConnection> nodeConn;
-        
+
         for ( int retry = 0; retry < 2; retry++ ) {
             bool triedQuickCheck = false;
 
@@ -716,6 +760,18 @@ namespace mongo {
             warning() << "No primary detected for set " << _name << endl;
             scoped_lock lk( _lock );
             _master = -1;
+
+            if (checkAllSecondaries) {
+                for ( unsigned i = 0; i < _nodes.size(); i++ ) {
+                    if ( _nodes[i].ok ) {
+                        _failedChecks = 0;
+                        return;
+                    }
+                }
+                // None of the nodes are ok.
+                _failedChecks++;
+                log() << "All nodes for set " << _name << " are down. This has happened for " << _failedChecks << " checks in a row. Polling will stop after " << _maxFailedChecks - _failedChecks << " more failed checks" << endl;
+            }
         }
     }
 
@@ -933,36 +989,57 @@ namespace mongo {
 
     mongo::mutex ReplicaSetMonitor::_setsLock( "ReplicaSetMonitor" );
     map<string,ReplicaSetMonitorPtr> ReplicaSetMonitor::_sets;
+    map<string,vector<HostAndPort> > ReplicaSetMonitor::_setServers;
     ReplicaSetMonitor::ConfigChangeHook ReplicaSetMonitor::_hook;
+    int ReplicaSetMonitor::_maxFailedChecks = 30; // At 1 check every 10 seconds, 30 checks takes 5 minutes
 
     // --------------------------------
     // ----- DBClientReplicaSet ---------
     // --------------------------------
 
     DBClientReplicaSet::DBClientReplicaSet( const string& name , const vector<HostAndPort>& servers, double so_timeout )
-        : _monitor( ReplicaSetMonitor::get( name , servers ) ),
-          _so_timeout( so_timeout ) {
+        : _setName( name ), _so_timeout( so_timeout ) {
+        ReplicaSetMonitor::createIfNeeded( name, servers );
     }
 
     DBClientReplicaSet::~DBClientReplicaSet() {
     }
 
+    ReplicaSetMonitorPtr DBClientReplicaSet::_getMonitor() const {
+        ReplicaSetMonitorPtr rsm = ReplicaSetMonitor::get( _setName, true );
+        // If you can't get a ReplicaSetMonitor then this connection isn't valid
+        uassert( 16340, str::stream() << "No replica set monitor active and no cached seed "
+                 "found for set: " << _setName, rsm );
+        return rsm;
+    }
+
+    // This can't throw an exception because it is called in the destructor of ScopedDbConnection
+    string DBClientReplicaSet::getServerAddress() const {
+        ReplicaSetMonitorPtr rsm = ReplicaSetMonitor::get( _setName, true );
+        if ( !rsm ) {
+            warning() << "Trying to get server address for DBClientReplicaSet, but no "
+                "ReplicaSetMonitor exists for " << _setName << endl;
+            return str::stream() << _setName << "/" ;
+        }
+        return rsm->getServerAddress();
+    }
+
     DBClientConnection * DBClientReplicaSet::checkMaster() {
-        HostAndPort h = _monitor->getMaster();
+        ReplicaSetMonitorPtr monitor = _getMonitor();
+        HostAndPort h = monitor->getMaster();
 
         if ( h == _masterHost && _master ) {
             // a master is selected.  let's just make sure connection didn't die
             if ( ! _master->isFailed() )
                 return _master.get();
-
-            _monitor->notifyFailure( _masterHost );
+            monitor->notifyFailure( _masterHost );
         }
 
-        _masterHost = _monitor->getMaster();
+        _masterHost = monitor->getMaster();
         _master.reset( new DBClientConnection( true , this , _so_timeout ) );
         string errmsg;
         if ( ! _master->connect( _masterHost , errmsg ) ) {
-            _monitor->notifyFailure( _masterHost );
+            monitor->notifyFailure( _masterHost );
             uasserted( 13639 , str::stream() << "can't connect to new replica set master [" << _masterHost.toString() << "] err: " << errmsg );
         }
         _auth( _master.get() );
@@ -970,14 +1047,15 @@ namespace mongo {
     }
 
     DBClientConnection * DBClientReplicaSet::checkSlave() {
-        HostAndPort h = _monitor->getSlave( _slaveHost );
+        ReplicaSetMonitorPtr monitor = _getMonitor();
+        HostAndPort h = monitor->getSlave( _slaveHost );
 
         if ( h == _slaveHost && _slave ) {
             if ( ! _slave->isFailed() )
                 // TODO: SERVER-5082, slave auth credentials may have changed
                 return _slave.get();
-            _monitor->notifySlaveFailure( _slaveHost );
-            _slaveHost = _monitor->getSlave();
+            monitor->notifySlaveFailure( _slaveHost );
+            _slaveHost = monitor->getSlave();
         } 
         else {
             _slaveHost = h;
@@ -995,7 +1073,7 @@ namespace mongo {
             const AuthInfo& a = *i;
             string errmsg;
             if ( ! conn->auth( a.dbname , a.username , a.pwd , errmsg, a.digestPassword ) )
-                warning() << "cached auth failed for set: " << _monitor->getName() << " db: " << a.dbname << " user: " << a.username << endl;
+                warning() << "cached auth failed for set: " << _setName << " db: " << a.dbname << " user: " << a.username << endl;
 
         }
     }
@@ -1013,8 +1091,11 @@ namespace mongo {
             checkMaster();
         }
         catch (AssertionException&) {
-            if (_master && _monitor) {
-                _monitor->notifyFailure(_masterHost);
+            // Can't use _getMonitor because that will create a new monitor from the cached seed if
+            // the monitor doesn't exist.
+            ReplicaSetMonitorPtr monitor = ReplicaSetMonitor::get(_setName);
+            if (_master && monitor ) {
+                monitor->notifyFailure(_masterHost);
             }
             return false;
         }
@@ -1099,7 +1180,12 @@ namespace mongo {
 
     void DBClientReplicaSet::isntMaster() { 
         log() << "got not master for: " << _masterHost << endl;
-        _monitor->notifyFailure( _masterHost );
+        // Can't use _getMonitor because that will create a new monitor from the cached seed if
+        // the monitor doesn't exist.
+        ReplicaSetMonitorPtr monitor = ReplicaSetMonitor::get( _setName );
+        if ( monitor ) {
+            monitor->notifyFailure( _masterHost );
+        }
         _master.reset(); 
     }
 
@@ -1125,7 +1211,7 @@ namespace mongo {
     void DBClientReplicaSet::isntSecondary() {
         log() << "slave no longer has secondary status: " << _slaveHost << endl;
         // Failover to next slave
-        _monitor->notifySlaveFailure( _slaveHost );
+        _getMonitor()->notifySlaveFailure( _slaveHost );
         _slave.reset();
     }
 
diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h
index 78bcfa0020a..bf2050ef22f 100644
--- a/src/mongo/client/dbclient_rs.h
+++ b/src/mongo/client/dbclient_rs.h
@@ -119,6 +119,7 @@ namespace mongo {
             bool hidden;
 
             int pingTimeMillis;
+
         };
 
         /**
@@ -145,14 +146,16 @@ namespace mongo {
                 int& nextNodeIndex );
 
         /**
-         * gets a cached Monitor per name or will create if doesn't exist
+         * Creates a new ReplicaSetMonitor, if it doesn't already exist.
          */
-        static ReplicaSetMonitorPtr get( const string& name , const vector<HostAndPort>& servers );
+        static void createIfNeeded( const string& name , const vector<HostAndPort>& servers );
 
         /**
-         * gets a cached Monitor per name or will return none if it doesn't exist
+         * gets a cached Monitor per name. If the monitor is not found and createFromSeed is false,
+         * it will return none. If createFromSeed is true, it will try to look up the last known
+         * servers list for this set and will create a new monitor using that as the seed list.
          */
-        static ReplicaSetMonitorPtr get( const string& name );
+        static ReplicaSetMonitorPtr get( const string& name, const bool createFromSeed = false );
 
 
         /**
@@ -162,9 +165,14 @@ namespace mongo {
         static void checkAll( bool checkAllSecondaries );
 
         /**
-         * deletes the ReplicaSetMonitor for the given set name.
+         * Removes the ReplicaSetMonitor for the given set name from _sets, which will delete it.
+         * If clearSeedCache is true, then the cached seed string for this Replica Set will be removed
+         * from _setServers.
          */
-        static void remove( const string& name );
+        static void remove( const string& name, bool clearSeedCache = false );
+
+        static int getMaxFailedChecks() { return _maxFailedChecks; };
+        static void setMaxFailedChecks(int numChecks) { _maxFailedChecks = numChecks; };
 
         /**
          * this is called whenever the config of any replica set changes
@@ -221,12 +229,14 @@ namespace mongo {
     private:
         /**
          * This populates a list of hosts from the list of seeds (discarding the
-         * seed list).
+         * seed list). Should only be called from within _setsLock.
          * @param name set name
          * @param servers seeds
          */
         ReplicaSetMonitor( const string& name , const vector<HostAndPort>& servers );
 
+        static void _remove_inlock( const string& name, bool clearSeedCache = false );
+
         /**
          * Checks all connections from the host list and sets the current
          * master.
@@ -266,6 +276,13 @@ namespace mongo {
         bool _checkConnection( DBClientConnection* conn, string& maybePrimary,
                 bool verbose, int nodesOffset );
 
+        /**
+         * Save the seed list for the current set into the _setServers map
+         * Should only be called if you're already holding _setsLock and this
+         * monitor's _lock.
+         */
+        void _cacheServerAddresses_inlock();
+
         string _getServerAddress_inlock() const;
 
         NodeDiff _getHostDiff_inlock( const BSONObj& hostList );
@@ -335,11 +352,17 @@ namespace mongo {
 
         int _master; // which node is the current master.  -1 means no master is known
         int _nextSlave; // which node is the current slave
+        // The number of consecutive times the set has been checked and every member in the set was down.
+        int _failedChecks;
 
-        static mongo::mutex _setsLock; // protects _sets
+        static mongo::mutex _setsLock; // protects _sets and _setServers
         static map<string,ReplicaSetMonitorPtr> _sets; // set name to Monitor
+        static map<string,vector<HostAndPort> > _setServers; // set name to seed list. Used to rebuild the monitor if it is cleaned up but then the set is accessed again.
+
         static ConfigChangeHook _hook;
         int _localThresholdMillis; // local ping latency threshold (protected by _lock)
+
+        static int _maxFailedChecks;
     };
 
     /** Use this class to connect to a replica set of servers.  The class will manage
@@ -421,7 +444,7 @@ namespace mongo {
 
         string toString() { return getServerAddress(); }
 
-        string getServerAddress() const { return _monitor->getServerAddress(); }
+        string getServerAddress() const;
 
         virtual ConnectionString::ConnectionType type() const { return ConnectionString::SET; }
         virtual bool lazySupported() const { return true; }
@@ -445,7 +468,10 @@ namespace mongo {
 
         void _auth( DBClientConnection * conn );
 
-        ReplicaSetMonitorPtr _monitor;
+        // Throws a DBException if the monitor doesn't exist and there isn't a cached seed to use.
+        ReplicaSetMonitorPtr _getMonitor() const;
+
+        string _setName;
 
         HostAndPort _masterHost;
         scoped_ptr<DBClientConnection> _master;
diff --git a/src/mongo/db/dbcommands_generic.cpp b/src/mongo/db/dbcommands_generic.cpp
index 2e85a4efb80..0cef5762a82 100644
--- a/src/mongo/db/dbcommands_generic.cpp
+++ b/src/mongo/db/dbcommands_generic.cpp
@@ -22,6 +22,7 @@
 #include <time.h>
 #include "introspect.h"
 #include "btree.h"
+#include "../client/dbclient_rs.h"
 #include "../util/lruishmap.h"
 #include "../util/md5.hpp"
 #include "../util/processinfo.h"
@@ -243,6 +244,11 @@ namespace mongo {
                 DBException::traceExceptions = cmdObj["traceExceptions"].Bool();
                 s++;
             }
+            if( cmdObj.hasElement( "replMonitorMaxFailedChecks" ) ) {
+                if( s == 0 ) result.append( "was", ReplicaSetMonitor::getMaxFailedChecks() );
+                ReplicaSetMonitor::setMaxFailedChecks( cmdObj["replMonitorMaxFailedChecks"].Number() );
+                s++;
+            }
 
             if( s == 0 && !found ) {
                 errmsg = "no option found to set, use help:true to see options ";
diff --git a/src/mongo/s/commands_admin.cpp b/src/mongo/s/commands_admin.cpp
index 6cc8dccd5a9..fa5fe501f72 100644
--- a/src/mongo/s/commands_admin.cpp
+++ b/src/mongo/s/commands_admin.cpp
@@ -1008,6 +1008,7 @@ namespace mongo {
                     }
 
                     Shard::removeShard( shardDoc[ "_id" ].str() );
+                    ReplicaSetMonitor::remove( shardDoc[ "_id" ].str(), true );
                     Shard::reloadShardInfo();
 
                     result.append( "msg"   , "removeshard completed successfully" );
diff --git a/src/mongo/s/config.h b/src/mongo/s/config.h
index 7ebd0cd02c6..d99484c9db3 100644
--- a/src/mongo/s/config.h
+++ b/src/mongo/s/config.h
@@ -25,6 +25,7 @@
 
 #include "../db/namespace.h"
 #include "../client/model.h"
+#include "mongo/client/dbclient_rs.h"
 
 #include "chunk.h"
 #include "shard.h"
diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp
index 3a4a956ddff..4a3e8fcc1dc 100644
--- a/src/mongo/s/shard.cpp
+++ b/src/mongo/s/shard.cpp
@@ -22,6 +22,7 @@
 #include "request.h"
 #include "client_info.h"
 #include "../db/commands.h"
+#include "mongo/client/dbclient_rs.h"
 #include "mongo/client/dbclientcursor.h"
 #include <set>
 
@@ -263,15 +264,6 @@ namespace mongo {
         _addr = addr;
         if ( !_addr.empty() ) {
             _cs = ConnectionString( addr , ConnectionString::SET );
-            _rsInit();
-        }
-    }
-
-    void Shard::_rsInit() {
-        if ( _cs.type() == ConnectionString::SET ) {
-            string x = _cs.getSetName();
-            massert( 14807 , str::stream() << "no set name for shard: " << _name << " " << _cs.toString() , x.size() );
-            _rs = ReplicaSetMonitor::get( x , _cs.getServers() );
         }
     }
 
@@ -279,22 +271,21 @@ namespace mongo {
         verify( _name.size() );
         _addr = cs.toString();
         _cs = cs;
-        _rsInit();
         staticShardInfo.set( _name , *this , true , false );
     }
 
     void Shard::reset( const string& ident ) {
         *this = staticShardInfo.findCopy( ident );
-        _rs.reset();
-        _rsInit();
     }
 
     bool Shard::containsNode( const string& node ) const {
         if ( _addr == node )
             return true;
 
-        if ( _rs && _rs->contains( node ) )
-            return true;
+        if ( _cs.type() == ConnectionString::SET ) {
+            ReplicaSetMonitorPtr rs = ReplicaSetMonitor::get( _cs.getSetName(), true );
+            return rs->contains( node );
+        }
 
         return false;
     }
diff --git a/src/mongo/s/shard.h b/src/mongo/s/shard.h
index 781fef46762..7dab578993f 100644
--- a/src/mongo/s/shard.h
+++ b/src/mongo/s/shard.h
@@ -21,7 +21,6 @@
 #include "pch.h"
 
 #include "mongo/client/connpool.h"
-#include "mongo/client/dbclient_rs.h"
 
 namespace mongo {
 
@@ -49,12 +48,12 @@ namespace mongo {
 
         Shard( const Shard& other )
             : _name( other._name ) , _addr( other._addr ) , _cs( other._cs ) , 
-              _maxSize( other._maxSize ) , _isDraining( other._isDraining ) , _rs( other._rs ) {
+              _maxSize( other._maxSize ) , _isDraining( other._isDraining ) {
         }
 
         Shard( const Shard* other )
             : _name( other->_name ) , _addr( other->_addr ), _cs( other->_cs ) , 
-              _maxSize( other->_maxSize ) , _isDraining( other->_isDraining ) , _rs( other->_rs ) {
+              _maxSize( other->_maxSize ) , _isDraining( other->_isDraining ) {
         }
 
         static Shard make( const string& ident ) {
@@ -160,8 +159,7 @@ namespace mongo {
         static Shard EMPTY;
         
     private:
-        
-	void _rsInit();
+
         void _setAddr( const string& addr );
         
         string    _name;
@@ -169,7 +167,6 @@ namespace mongo {
         ConnectionString _cs;
         long long _maxSize;    // in MBytes, 0 is unlimited
         bool      _isDraining; // shard is currently being removed
-        ReplicaSetMonitorPtr _rs;
     };
 
     class ShardStatus {