SERVER-17820: Handle long running exitCleanly in Service Stop

1 files changed, 30 insertions, 5 deletions

diff --git a/src/mongo/util/ntservice.cpp b/src/mongo/util/ntservice.cpp
index 8e8b66a2179..7aafb2681fb 100644
--- a/src/mongo/util/ntservice.cpp
+++ b/src/mongo/util/ntservice.cpp
@@ -514,6 +514,34 @@ namespace {
         return SetServiceStatus( _statusHandle, &ssStatus );
     }
 
+    static void serviceStopWorker() {
+        Client::initThread("serviceStopWorker");
+
+        // Stop the process
+        // TODO: SERVER-5703, separate the "cleanup for shutdown" functionality from
+        // the "terminate process" functionality in exitCleanly.
+        exitCleanly(EXIT_WINDOWS_SERVICE_STOP);
+    }
+
+    // Minimum of time we tell Windows to wait before we are guilty of a hung shutdown
+    const int kStopWaitHintMillis = 30000;
+
+    // Run exitCleanly on a separate thread so we can report progress to Windows
+    // Note: Windows may still kill us for taking too long,
+    // On client OSes, SERVICE_CONTROL_SHUTDOWN has a 5 second timeout configured in
+    // HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control
+    static void serviceStop() {
+        boost::thread serviceWorkerThread(serviceStopWorker);
+
+        // We periodically check if we are done exiting by polling at half of each wait interval
+        //
+        while (!serviceWorkerThread.try_join_for(
+            boost::chrono::milliseconds(kStopWaitHintMillis / 2))) {
+            reportStatus(SERVICE_STOP_PENDING, kStopWaitHintMillis);
+            log() << "Service Stop is waiting for storage engine to finish shutdown";
+        }
+    }
+
     static void WINAPI initService( DWORD argc, LPTSTR *argv ) {
         _statusHandle = RegisterServiceCtrlHandler( _serviceName.c_str(), serviceCtrl );
         if ( !_statusHandle )
@@ -523,14 +551,11 @@ namespace {
 
         ExitCode exitCode = _serviceCallback();
 
-        // Stop the process
         // During clean shutdown, ie NT SCM signals us, _serviceCallback returns here
         // as part of the listener loop terminating.
         // exitCleanly is supposed to return. If it blocks, some other thread must be exiting.
         //
-        // TODO: SERVER-5703, separate the "cleanup for shutdown" functionality from
-        // the "terminate process" functionality in exitCleanly.
-        exitCleanly( EXIT_WINDOWS_SERVICE_STOP );
+        serviceStop();
 
         reportStatus(SERVICE_STOPPED, 0, exitCode);
     }
@@ -541,7 +566,7 @@ namespace {
         log() << "got " << controlCodeName << " request from Windows Service Control Manager, " <<
             ( inShutdown() ? "already in shutdown" : "will terminate after current cmd ends" );
 
-        reportStatus( SERVICE_STOP_PENDING );
+        reportStatus(SERVICE_STOP_PENDING, kStopWaitHintMillis);
 
         // Note: This triggers _serviceCallback, ie  ServiceMain,
         // to stop by setting inShutdown() == true