SERVER-10015 balancer should stop when ConfigServerCheck indicates inconsistency

Conflicts:
	src/mongo/s/server.cpp

5 files changed, 105 insertions, 9 deletions

diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 27067e32e9d..816a6bfacf8 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -479,6 +479,7 @@ mongosLibraryFiles = [
     "s/commands_public.cpp",
     "s/request.cpp",
     "s/client_info.cpp",
+    "s/config_server_checker_service.cpp",
     "s/cursors.cpp",
     "s/s_only.cpp",
     "s/balance.cpp",
diff --git a/src/mongo/s/balance.cpp b/src/mongo/s/balance.cpp
index de72b25dada..2cf9d06d131 100644
--- a/src/mongo/s/balance.cpp
+++ b/src/mongo/s/balance.cpp
@@ -24,6 +24,7 @@
 #include "mongo/db/jsobj.h"
 #include "mongo/s/chunk.h"
 #include "mongo/s/config.h"
+#include "mongo/s/config_server_checker_service.h"
 #include "mongo/s/grid.h"
 #include "mongo/s/server.h"
 #include "mongo/s/shard.h"
@@ -461,7 +462,14 @@ namespace mongo {
                         sleepsecs( sleepTime ); // no need to wake up soon
                         continue;
                     }
-                    
+
+                    if ( !isConfigServerConsistent() ) {
+                        conn.done();
+                        warning() << "Skipping balancing round because data inconsistency"
+                                  << " was detected amongst the config servers." << endl;
+                        continue;
+                    }
+
                     LOG(1) << "*** start balancing round" << endl;
 
                     bool waitForDelete = false;
diff --git a/src/mongo/s/config_server_checker_service.cpp b/src/mongo/s/config_server_checker_service.cpp
new file mode 100644
index 00000000000..4d30cdcf50f
--- /dev/null
+++ b/src/mongo/s/config_server_checker_service.cpp
@@ -0,0 +1,62 @@
+/**
+ *    Copyright (C) 2013 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/thread/thread.hpp>
+
+#include "mongo/s/config.h"
+#include "mongo/s/config_server_checker_service.h"
+#include "mongo/util/concurrency/mutex.h"
+
+namespace mongo {
+
+    namespace {
+
+        // Thread that runs dbHash on config servers for checking data consistency.
+        boost::scoped_ptr<boost::thread> _checkerThread;
+
+        // Protects _isConsistentFromLastCheck.
+        mutex _isConsistentMutex( "ConfigServerConsistent" );
+        bool _isConsistentFromLastCheck = true;
+
+        void checkConfigConsistency() {
+            while ( !inShutdown() ) {
+                bool isConsistent = configServer.ok( true );
+
+                {
+                    scoped_lock sl( _isConsistentMutex );
+                    _isConsistentFromLastCheck = isConsistent;
+                }
+
+                sleepsecs( 60 );
+            }
+        }
+    }
+
+    bool isConfigServerConsistent() {
+        scoped_lock sl( _isConsistentMutex );
+        return _isConsistentFromLastCheck;
+    }
+
+    bool startConfigServerChecker() {
+        if ( _checkerThread == NULL ) {
+            _checkerThread.reset( new boost::thread( checkConfigConsistency ));
+        }
+
+        return _checkerThread != NULL;
+    }
+}
+
diff --git a/src/mongo/s/config_server_checker_service.h b/src/mongo/s/config_server_checker_service.h
new file mode 100644
index 00000000000..4661d0fc756
--- /dev/null
+++ b/src/mongo/s/config_server_checker_service.h
@@ -0,0 +1,31 @@
+/**
+ *    Copyright (C) 2013 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace mongo {
+
+    /**
+     * Returns true if the config servers have the same contents since the last check
+     * was performed. Currently checks only the config.chunks and config.databases.
+     */
+    bool isConfigServerConsistent();
+
+    /**
+     * Starts the thread that periodically checks data consistency amongst the config servers.
+     * Note: this is not thread safe.
+     */
+    bool startConfigServerChecker();
+}
+
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index 6f5f4ed52f7..e6aa728d7a3 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -45,6 +45,7 @@
 #include "cursors.h"
 #include "../util/processinfo.h"
 #include "mongo/db/lasterror.h"
+#include "mongo/s/config_server_checker_service.h"
 #include "mongo/s/config_upgrade.h"
 #include "mongo/util/stacktrace.h"
 #include "mongo/util/exception_filter_win32.h"
@@ -292,14 +293,7 @@ static bool runMongosServer( bool doUpgrade ) {
         return false;
     }
 
-    {
-        class CheckConfigServers : public task::Task {
-            virtual string name() const { return "CheckConfigServers"; }
-            virtual void doWork() { configServer.ok(true); }
-        };
-
-        task::repeat(new CheckConfigServers, 60*1000);
-    }
+    startConfigServerChecker();
 
     VersionType initVersionInfo;
     VersionType versionInfo;