SERVER-2213 balancer tolerates servers being down at mongos startup

2 files changed, 84 insertions, 36 deletions

diff --git a/s/balance.cpp b/s/balance.cpp
index 9d6391492fa..6e4a922b1e9 100644
--- a/s/balance.cpp
+++ b/s/balance.cpp
@@ -1,4 +1,4 @@
-// balance.cpp
+//@file balance.cpp
 
 /**
 *    Copyright (C) 2008 10gen Inc.
@@ -81,19 +81,6 @@ namespace mongo {
         return movedCount;
     }
     
-    void Balancer::_ping(){
-        assert( _myid.size() && _started );
-        try {
-            ScopedDbConnection conn( configServer.getPrimary() );
-            _ping( conn.conn() );
-            conn.done();
-        }
-        catch ( std::exception& e ){
-            log() << "bare ping failed: " << e.what() << endl;
-        }
-        
-    }
-
     void Balancer::_ping( DBClientBase& conn ){
         WriteConcern w = conn.getWriteConcern();
         conn.setWriteConcern( W_NONE );
@@ -219,22 +206,53 @@ namespace mongo {
         }
     }
 
-    void Balancer::run(){
+    bool Balancer::_init() {
+        try {
+            
+            log() << "about to contact config servers and shards" << endl;
+
+            // contact the config server and refresh shard information
+            // checks that each shard is indeed a different process (no hostname mixup)
+            // these checks are redundant in that they're redone at every new round but we want to do them initially here
+            // so to catch any problem soon
+            Shard::reloadShardInfo();
+            _checkOIDs();
+
+            log() << "config servers and shards contacted successfully" << endl;
 
-        { // init stuff, don't want to do at static init
             StringBuilder buf;
             buf << getHostNameCached() << ":" << cmdLine.port;
             _myid = buf.str();
-            log() << "balancer myid: " << _myid << endl;
-            
             _started = time(0);
 
-            Shard::reloadShardInfo();
+            log() << "balancer id: " << _myid << " started at " << time_t_to_String_short(_started) << endl;            
+
+            return true;
+
+        } catch ( std::exception& e ) {
+
+            log( LL_WARNING ) << "could not initialize balancer, please check that all shards and config servers are up" << endl;
+            return false;
+
+        }
+    }
+
+    void Balancer::run(){
+
+        // this is the body of a BackgroundJob so if we throw here we're basically ending the balancer thread prematurely
+        while ( ! inShutdown() ) {
+
+            if ( ! _init() ) {
+                log() << "will retry to initialize balancer in one minute" << endl;
+                sleepsecs( 60 );
+                continue;
+            }
+
+            break;
         }
-        
-        _ping();
-        _checkOIDs();
 
+        // getConnectioString and the constructor of a DistributedLock do not throw, which is what we expect on while
+        // on the balancer thread
         ConnectionString config = configServer.getConnectionString();
         DistributedLock balanceLock( config , "balancer" );
 
diff --git a/s/balance.h b/s/balance.h
index 1d095752e22..80662b5c761 100644
--- a/s/balance.h
+++ b/s/balance.h
@@ -1,4 +1,4 @@
-// balance.h
+//@file balance.h
 
 /**
 *    Copyright (C) 2008 10gen Inc.
@@ -25,6 +25,15 @@
 
 namespace mongo {
     
+    /**
+     * The balancer is a background task that tries to keep the number of chunks across all servers of the cluster even. Although
+     * every mongos will have one balancer running, only one of them will be active at the any given point in time. The balancer
+     * uses a 'DistributedLock' for that coordination.
+     *
+     * The balancer does act continuously but in "rounds". At a given round, it would decide if there is an imbalance by 
+     * checking the difference in chunks between the most and least loaded shards. It would issue a request for a chunk 
+     * migration per round, if it found so.
+     */
     class Balancer : public BackgroundJob {
     public:
         Balancer();
@@ -40,35 +49,56 @@ namespace mongo {
         typedef BalancerPolicy::ChunkInfo CandidateChunk;
         typedef shared_ptr<CandidateChunk> CandidateChunkPtr;
 
+        // hostname:port of my mongos
+        string _myid;
+
+        // time the Balancer started running
+        time_t _started;
+
+        // number of moved chunks in last round
+        int _balancedLastTime; 
+
+        // decide which chunks to move; owned here.
+        BalancerPolicy* _policy;           
+
         /**
-         * Gathers all the necessary information about shards and chunks, and 
-         * decides whether there are candidate chunks to be moved.
+         * Checks that the balancer can connect to all servers it needs to do its job.
+         *
+         * @return true if balancing can be started
+         *
+         * This method throws on a network exception
+         */
+        bool _init();
+        
+        /**
+         * Gathers all the necessary information about shards and chunks, and decides whether there are candidate chunks to
+         * be moved.
+         *
+         * @param conn is the connection with the config server(s)
+         * @param candidateChunks (IN/OUT) filled with candidate chunks, one per collection, that could possibly be moved
          */
         void _doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks );
 
         /**
-         * Execute the chunk migrations described in 'candidateChunks' and
-         * returns the number of chunks effectively moved.
+         * Issues chunk migration request, one at a time.
+         * 
+         * @param candidateChunks possible chunks to move
+         * @return number of chunks effectively moved
          */
         int _moveChunks( const vector<CandidateChunkPtr>* candidateChunks );
 
         /**
-         * Check the health of the master configuration server
+         * Marks this balancer as being live on the config server(s).
+         *
+         * @param conn is the connection with the config server(s)
          */
-        void _ping();
         void _ping( DBClientBase& conn );
 
         /**
-         * @return true if everything is ok
+         * @return true if all the servers listed in configdb as being shards are reachable and are distinct processes
          */
         bool _checkOIDs();
 
-        // internal state
-
-        string          _myid;             // hostname:port of my mongos
-        time_t          _started;          // time Balancer starte running
-        int             _balancedLastTime; // number of moved chunks in last round
-        BalancerPolicy* _policy;           // decide which chunks to move; owned here.
     };
     
     extern Balancer balancer;