SERVER-15949:add diagnostics to network executor

6 files changed, 65 insertions, 4 deletions

diff --git a/src/mongo/db/repl/network_interface_impl.cpp b/src/mongo/db/repl/network_interface_impl.cpp
index 7c838f4e7c5..483d7a29487 100644
--- a/src/mongo/db/repl/network_interface_impl.cpp
+++ b/src/mongo/db/repl/network_interface_impl.cpp
@@ -337,13 +337,28 @@ namespace {
 
     NetworkInterfaceImpl::NetworkInterfaceImpl() :
         _isExecutorRunnable(false),
-        _inShutdown(false) {
+        _inShutdown(false),
+        _numActiveNetworkRequests(0) {
         ConnectionPool::Options options;
         _connPool.reset(new ConnectionPool(options));
     }
 
     NetworkInterfaceImpl::~NetworkInterfaceImpl() { }
 
+    std::string NetworkInterfaceImpl::getDiagnosticString() {
+        boost::lock_guard<boost::mutex> lk(_mutex);
+        str::stream output;
+        output << "NetworkImpl";
+        output << " threads:" << _threads.size();
+        output << " inShutdown:" << _inShutdown;
+        output << " active:" << _numActiveNetworkRequests;
+        output << " pending:" << _pending.size();
+        output << " execRunable:" << _isExecutorRunnable;
+        return output;
+
+    }
+
+
     void NetworkInterfaceImpl::startup() {
         boost::lock_guard<boost::mutex> lk(_mutex);
         invariant(!_inShutdown);
@@ -415,12 +430,14 @@ namespace {
             }
             CommandData todo = _pending.front();
             _pending.pop_front();
+            ++_numActiveNetworkRequests;
             lk.unlock();
             ResponseStatus result = _runCommand(todo.request);
             LOG(2) << "Network status of sending " << todo.request.cmdObj.firstElementFieldName() <<
                 " to " << todo.request.target << " was " << result.getStatus();
             todo.onFinish(result);
             lk.lock();
+            --_numActiveNetworkRequests;
             _signalWorkAvailable_inlock();
         }
     }
@@ -477,7 +494,7 @@ namespace {
                     return StatusWith<int>(ErrorCodes::ExceededTimeLimit,
                                                str::stream() << "Went to run command,"
                                                " but it was too late. Expiration was set to "
-                                                             << expDate);
+                                                             << dateToISOStringUTC(expDate));
                 }
             }
             return StatusWith<int>(timeout);
diff --git a/src/mongo/db/repl/network_interface_impl.h b/src/mongo/db/repl/network_interface_impl.h
index aa5c6955ead..717bfa8a5ae 100644
--- a/src/mongo/db/repl/network_interface_impl.h
+++ b/src/mongo/db/repl/network_interface_impl.h
@@ -49,6 +49,7 @@ namespace repl {
     public:
         explicit NetworkInterfaceImpl();
         virtual ~NetworkInterfaceImpl();
+        virtual std::string getDiagnosticString();
         virtual void startup();
         virtual void shutdown();
         virtual void waitForWork();
@@ -122,6 +123,8 @@ namespace repl {
         // requests.
         boost::scoped_ptr<ConnectionPool> _connPool;  // (R)
 
+        // Number of active network requests
+        int _numActiveNetworkRequests;
     };
 
 }  // namespace repl
diff --git a/src/mongo/db/repl/network_interface_mock.cpp b/src/mongo/db/repl/network_interface_mock.cpp
index ac63ffbb9ca..f0fd79fe667 100644
--- a/src/mongo/db/repl/network_interface_mock.cpp
+++ b/src/mongo/db/repl/network_interface_mock.cpp
@@ -57,6 +57,11 @@ namespace repl {
         invariant(_blackHoled.empty());
     }
 
+    std::string NetworkInterfaceMock::getDiagnosticString() {
+        // TODO something better.
+        return "NetworkInterfaceMock diagnostics here";
+    }
+
     Date_t NetworkInterfaceMock::now() {
         boost::lock_guard<boost::mutex> lk(_mutex);
         return _now_inlock();
diff --git a/src/mongo/db/repl/network_interface_mock.h b/src/mongo/db/repl/network_interface_mock.h
index bd5e507b64c..fd2ad52746d 100644
--- a/src/mongo/db/repl/network_interface_mock.h
+++ b/src/mongo/db/repl/network_interface_mock.h
@@ -67,6 +67,7 @@ namespace repl {
 
         NetworkInterfaceMock();
         virtual ~NetworkInterfaceMock();
+        virtual std::string getDiagnosticString();
 
         ////////////////////////////////////////////////////////////////////////////////
         //
diff --git a/src/mongo/db/repl/replication_executor.cpp b/src/mongo/db/repl/replication_executor.cpp
index 8a8298142fe..ad24282b404 100644
--- a/src/mongo/db/repl/replication_executor.cpp
+++ b/src/mongo/db/repl/replication_executor.cpp
@@ -58,6 +58,26 @@ namespace {
 
     ReplicationExecutor::~ReplicationExecutor() {}
 
+    std::string ReplicationExecutor::getDiagnosticString() {
+        boost::lock_guard<boost::mutex> lk(_mutex);
+        return _getDiagnosticString_inlock();
+    }
+
+    std::string ReplicationExecutor::_getDiagnosticString_inlock() const {
+        str::stream output;
+        output << "ReplicationExecutor";
+        output << " networkInProgress:" << _networkInProgressQueue.size();
+        output << " exclusiveInProgress:" << _exclusiveLockInProgressQueue.size();
+        output << " sleeperQueue:" << _sleepersQueue.size();
+        output << " ready:" << _readyQueue.size();
+        output << " free:" << _freeQueue.size();
+        output << " unsignaledEvents:" << _unsignaledEvents.size();
+        output << " eventWaiters:" << _totalEventWaiters;
+        output << " shuttingDown:" << _inShutdown;
+        output << " networkInterface:" << _networkInterface->getDiagnosticString();
+        return output;
+    }
+
     Date_t ReplicationExecutor::now() {
         return _networkInterface->now();
     }
@@ -483,7 +503,7 @@ namespace {
         }
     }
 
-    std::string ReplicationExecutor::RemoteCommandRequest::toString() const {
+    std::string ReplicationExecutor::RemoteCommandRequest::getDiagnosticString() {
         str::stream out;
         out << "RemoteCommand -- target:" << target.toString() << " db:" << dbname;
 
diff --git a/src/mongo/db/repl/replication_executor.h b/src/mongo/db/repl/replication_executor.h
index 8da5f466481..fcac23469b4 100644
--- a/src/mongo/db/repl/replication_executor.h
+++ b/src/mongo/db/repl/replication_executor.h
@@ -153,6 +153,11 @@ namespace repl {
         ~ReplicationExecutor();
 
         /**
+         * Returns diagnostic information.
+         */
+        std::string getDiagnosticString();
+
+        /**
          * Gets the current time as reported by the network interface.
          */
         Date_t now();
@@ -301,6 +306,10 @@ namespace repl {
         typedef stdx::list<Event> EventList;
 
         /**
+         * Returns diagnostic info
+         */
+        std::string _getDiagnosticString_inlock() const;
+        /**
          * Implementation of makeEvent() for use when _mutex is already held.
          */
         StatusWith<EventHandle> makeEvent_inlock();
@@ -459,7 +468,8 @@ namespace repl {
                              const BSONObj& theCmdObj,
                              const Milliseconds timeoutMillis = kNoTimeout);
 
-        std::string toString() const;
+        // Returns diagnostic info.
+        std::string getDiagnosticString();
 
         HostAndPort target;
         std::string dbname;
@@ -490,6 +500,11 @@ namespace repl {
         virtual ~NetworkInterface();
 
         /**
+         * Returns diagnostic info.
+         */
+        virtual std::string getDiagnosticString() = 0;
+
+        /**
          * Starts up the network interface.
          *
          * It is valid to call all methods except shutdown() before this method completes.  That is,