SERVER-13306 mongobridge should not try forever to connect to dead

destinations.

The connect timeout is set to 15 seconds of retrying, after which the
connection to the client will be terminated.

Also added some extra logging and diagnostics so it is easier to analyse
test failures in the future.

3 files changed, 27 insertions, 6 deletions

diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 8bd16cb921a..4a71174f611 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -753,7 +753,7 @@ ReplSetTest.prototype.stop = function( n , signal, wait /* wait for stop */, opt
         wait = signal
         signal = undefined
     }
-        
+    
     wait = wait || false
     if( ! wait.toFixed ){
         if( wait ) wait = 0
@@ -764,11 +764,15 @@ ReplSetTest.prototype.stop = function( n , signal, wait /* wait for stop */, opt
     print('ReplSetTest stop *** Shutting down mongod in port ' + port + ' ***');
     var ret = MongoRunner.stopMongod( port , signal, opts );
     
-    if( ! ret || wait < 0 ) return ret
+    if( ! ret || wait < 0 ) {
+        print('ReplSetTest stop *** Mongod in port ' + port + ' shutdown with code (' 
+                    + ret + '), wait (' + wait + ') ***');
+        return ret;
+    }
     
     // Wait for shutdown
     this.waitForHealth( n, this.DOWN, wait )
-    
+
     return true
 }
 
diff --git a/src/mongo/tools/bridge.cpp b/src/mongo/tools/bridge.cpp
index 0b75d5adab8..a648321508f 100644
--- a/src/mongo/tools/bridge.cpp
+++ b/src/mongo/tools/bridge.cpp
@@ -38,6 +38,7 @@
 #include "mongo/util/net/message.h"
 #include "mongo/util/stacktrace.h"
 #include "mongo/util/text.h"
+#include "mongo/util/timer.h"
 
 using namespace mongo;
 using namespace std;
@@ -48,11 +49,26 @@ class Forwarder {
 public:
     Forwarder( MessagingPort &mp ) : mp_( mp ) {
     }
+
     void operator()() const {
         DBClientConnection dest;
         string errmsg;
-        while (!dest.connect(mongoBridgeGlobalParams.destUri, errmsg))
-            sleepmillis( 500 );
+
+        Timer connectTimer;
+        while (!dest.connect(mongoBridgeGlobalParams.destUri, errmsg)) {
+            // If we can't connect for the configured timeout, give up
+            //
+            if (connectTimer.seconds() >= mongoBridgeGlobalParams.connectTimeoutSec) {
+                cout << "Unable to establish connection from " << mp_.psock->remoteString() 
+                     << " to " << mongoBridgeGlobalParams.destUri 
+                     << " after " << connectTimer.seconds() << " seconds. Giving up." << endl;
+                mp_.shutdown();
+                return;
+            }
+
+            sleepmillis(500);
+        }
+
         Message m;
         while( 1 ) {
             try {
diff --git a/src/mongo/tools/mongobridge_options.h b/src/mongo/tools/mongobridge_options.h
index 58e5685ab98..181bd273db4 100644
--- a/src/mongo/tools/mongobridge_options.h
+++ b/src/mongo/tools/mongobridge_options.h
@@ -46,9 +46,10 @@ namespace mongo {
     struct MongoBridgeGlobalParams {
         int port;
         int delay;
+        int connectTimeoutSec;
         string destUri;
 
-        MongoBridgeGlobalParams() : port(0), delay(0) { }
+        MongoBridgeGlobalParams() : port(0), delay(0), connectTimeoutSec(15) {}
     };
 
     extern MongoBridgeGlobalParams mongoBridgeGlobalParams;