SERVER-35946 Powercycle kill_mongod function should ensure the service is not in a running state

(cherry picked from commit b774421ee0c87ca64a10e8f5b360dd41471f4471)

1 files changed, 26 insertions, 8 deletions

diff --git a/pytests/powertest.py b/pytests/powertest.py
index 452905235a9..b6ca3902f76 100755
--- a/pytests/powertest.py
+++ b/pytests/powertest.py
@@ -904,13 +904,22 @@ class WindowsService(object):
 
         return ret, output
 
-    def stop(self):
-        """ Stop service. Returns (code, output) tuple. """
+    def stop(self, timeout):
+        """Stop service, waiting for 'timeout' seconds. Return (code, output) tuple."""
         self.pids = []
         if self.status() not in self._states.values():
             return 1, "Service '{}' status: {}".format(self.name, self.status())
         try:
             win32serviceutil.StopService(serviceName=self.name)
+            start = time.time()
+            status = self.status()
+            while status == "stop pending":
+                if time.time() - start >= timeout:
+                    ret = 1
+                    output = "Service '{}' status is '{}'".format(self.name, status)
+                    break
+                time.sleep(3)
+                status = self.status()
             ret = 0
             output = "Service '{}' stopped".format(self.name)
         except pywintypes.error as err:
@@ -972,8 +981,8 @@ class PosixService(object):
             self.pids = proc.get_pids()
         return ret, output
 
-    def stop(self):
-        """ Stop process. Returns (code, output) tuple. """
+    def stop(self, timeout):  # pylint: disable=unused-argument
+        """Stop process. Returns (code, output) tuple."""
         proc = ProcessControl(name=self.bin_name)
         proc.kill()
         self.pids = []
@@ -1083,9 +1092,9 @@ class MongodControl(object):
         """ Returns tuple (ret, ouput). """
         return self.service.update()
 
-    def stop(self):
-        """ Returns tuple (ret, ouput). """
-        return self.service.stop()
+    def stop(self, timeout=0):
+        """Return tuple (ret, ouput)."""
+        return self.service.stop(timeout)
 
     def status(self):
         """Return status of the process."""
@@ -1192,8 +1201,17 @@ def remote_handler(options, operations):
                 pass
 
         elif operation == "kill_mongod":
-            # Unconditional kill of mongod
+            # Unconditional kill of mongod.
             ret, output = kill_mongod()
+            if ret:
+                LOGGER.error("kill_mongod failed %s", output)
+                return ret
+            # Ensure the mongod service is not in a running state.
+            mongod.stop(timeout=30)
+            status = mongod.status()
+            if status != "stopped":
+                LOGGER.error("Unable to stop the mongod service, in state '%s'", status)
+                ret = 1
 
         elif operation == "install_mongod":
             ret, output = mongod.install(root_dir, options.tarball_url)