SERVER-35506 The Powercycle wait_for_mongod_shutdown function should ensure the mongod process is no longer running

1 files changed, 27 insertions, 12 deletions

diff --git a/pytests/powertest.py b/pytests/powertest.py
index 0c5757cd371..477a5f460b5 100755
--- a/pytests/powertest.py
+++ b/pytests/powertest.py
@@ -286,6 +286,13 @@ def get_extension(filename):
     return os.path.splitext(filename)[-1]
 
 
+def executable_extension():
+    """Return executable file extension."""
+    if _IS_WINDOWS:
+        return ".exe"
+    return ""
+
+
 def abs_path(path):
     """Return absolute path for 'path'. Raises an exception on failure."""
     if _IS_WINDOWS:
@@ -964,8 +971,7 @@ class MongodControl(object):  # pylint: disable=too-many-instance-attributes
     def __init__(  # pylint: disable=too-many-arguments
             self, bin_dir, db_path, log_path, port, options=None):
         """Initialize MongodControl."""
-        extension = ".exe" if _IS_WINDOWS else ""
-        self.process_name = "mongod{}".format(extension)
+        self.process_name = "mongod{}".format(executable_extension())
 
         self.bin_dir = bin_dir
         if self.bin_dir:
@@ -1057,6 +1063,10 @@ class MongodControl(object):  # pylint: disable=too-many-instance-attributes
         """Return tuple (ret, ouput)."""
         return self.service.stop()
 
+    def status(self):
+        """Return status of the process."""
+        return self.service.status()
+
     def get_pids(self):
         """Return list of pids for process."""
         return self.service.get_pids()
@@ -1184,7 +1194,7 @@ def remote_handler(options, operations):  # pylint: disable=too-many-branches,to
         elif operation == "stop_mongod":
             ret, output = mongod.stop()
             LOGGER.info(output)
-            ret = wait_for_mongod_shutdown(options.db_path)
+            ret = wait_for_mongod_shutdown(mongod)
 
         elif operation == "shutdown_mongod":
             mongo = pymongo.MongoClient(**mongo_client_opts)
@@ -1192,7 +1202,7 @@ def remote_handler(options, operations):  # pylint: disable=too-many-branches,to
                 mongo.admin.command("shutdown", force=True)
             except pymongo.errors.AutoReconnect:
                 pass
-            ret = wait_for_mongod_shutdown(options.db_path)
+            ret = wait_for_mongod_shutdown(mongod)
 
         elif operation == "rsync_data":
             ret, output = rsync(options.db_path, options.rsync_dest, options.rsync_exclude_files)
@@ -1367,18 +1377,23 @@ def crash_server_or_kill_mongod(  # pylint: disable=too-many-arguments,,too-many
     return ret, output
 
 
-def wait_for_mongod_shutdown(data_dir, timeout=120):
+def wait_for_mongod_shutdown(mongod_control, timeout=120):
     """Wait for for mongod to shutdown; return 0 if shutdown occurs within 'timeout', else 1."""
-
-    lock_file = os.path.join(data_dir, "mongod.lock")
-    LOGGER.info("Waiting for mongod to release lockfile %s", lock_file)
     start = time.time()
-    while os.path.exists(lock_file) and os.stat(lock_file).st_size:
-        time.sleep(3)
+    status = mongod_control.status()
+    while status != "stopped":
         if time.time() - start >= timeout:
-            LOGGER.error("The mongod lockfile %s has not been released, exiting", lock_file)
+            LOGGER.error("The mongod process has not stopped, current status is %s", status)
             return 1
-    LOGGER.info("The mongod lockfile %s has been released", lock_file)
+        LOGGER.info("Waiting for mongod process to stop, current status is %s ", status)
+        time.sleep(3)
+        status = mongod_control.status()
+    LOGGER.info("The mongod process has stopped")
+
+    # We wait a bit, since files could still be flushed to disk, which was causing
+    # rsync "file has vanished" errors.
+    time.sleep(5)
+
     return 0