SERVER-29816 Powercyle test added, pytests/powertest.py.

1 files changed, 1751 insertions, 0 deletions

diff --git a/pytests/powertest.py b/pytests/powertest.py
new file mode 100755
index 00000000000..49ce400d589
--- /dev/null
+++ b/pytests/powertest.py
@@ -0,0 +1,1751 @@
+#!/usr/bin/env python
+
+"""Powercycle test
+
+Tests robustness of mongod to survice multiple powercycle events.
+"""
+
+from __future__ import print_function
+
+import collections
+import copy
+import datetime
+import distutils.spawn
+import importlib
+import logging
+import optparse
+import os
+import random
+import re
+import shlex
+import shutil
+import stat
+import string
+import sys
+import tarfile
+import tempfile
+import time
+import urlparse
+import zipfile
+
+import psutil
+import pymongo
+import requests
+import yaml
+
+# The subprocess32 module is untested on Windows and thus isn't recommended for use, even when it's
+# installed. See https://github.com/google/python-subprocess32/blob/3.2.7/README.md#usage.
+if os.name == "posix" and sys.version_info[0] == 2:
+    try:
+        import subprocess32 as subprocess
+    except ImportError:
+        import warnings
+        warnings.warn(("Falling back to using the subprocess module because subprocess32 isn't"
+                       " available. When using the subprocess module, a child process may"
+                       " trigger an invalid free(). See SERVER-22219 for more details."),
+                      RuntimeWarning)
+        import subprocess
+else:
+    import subprocess
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+# See https://docs.python.org/2/library/sys.html#sys.platform
+_IS_WINDOWS = sys.platform == "win32" or sys.platform == "cygwin"
+_IS_LINUX = sys.platform.startswith("linux")
+_IS_DARWIN = sys.platform == "darwin"
+
+
+def _try_import(module, name=None):
+    """Attempts to import a module and add it as a global variable.
+       If the import fails, then this function doesn't trigger an exception."""
+    try:
+        module_name = module if not name else name
+        globals()[module_name] = importlib.import_module(module)
+    except ImportError:
+        pass
+
+
+# These modules are used on the 'client' side.
+_try_import("buildscripts.aws_ec2", "aws_ec2")
+_try_import("buildscripts.remote_operations", "remote_operations")
+
+if _IS_WINDOWS:
+
+    # These modules are used on the 'server' side.
+    _try_import("ntsecuritycon")
+    _try_import("pywintypes")
+    _try_import("win32file")
+    _try_import("win32security")
+    _try_import("win32service")
+    _try_import("win32serviceutil")
+
+
+__version__ = "0.1"
+
+LOGGER = logging.getLogger(__name__)
+
+""" Client & server side powercycle test script.
+
+    This script can be run against any host which is reachable via ssh.
+    Note - the remote hosts should be running bash shell (this script may fail otherwise).
+    There are no assumptions on the server what is the current deployment of MongoDB.
+    For Windows the assumption is that Cygwin is installed.
+    The server needs these utilities:
+        - python 2.7 or higher
+        - sshd
+        - rsync
+    This script will either download a MongoDB tarball or use an existing setup. """
+
+
+def get_extension(filename):
+    """Returns the extension of a file."""
+    return os.path.splitext(filename)[-1]
+
+
+def abs_path(path):
+    """Returns absolute path for 'path'. Raises an exception on failure."""
+    if _IS_WINDOWS:
+        # Get the Windows absolute path.
+        cmd = "cygpath -wa {}".format(path)
+        ret, output = execute_cmd(cmd, use_file=True)
+        if ret:
+            raise Exception("Command \"{}\" failed with code {} and output message: {}".format(
+                cmd, ret, output))
+        return output.rstrip()
+    return os.path.abspath(os.path.normpath(path))
+
+
+def symlink_dir(source_dir, dest_dir):
+    """Symlinks the 'dest_dir' to 'source_dir'."""
+    if _IS_WINDOWS:
+        win32file.CreateSymbolicLink(dest_dir, source_dir, win32file.SYMBOLIC_LINK_FLAG_DIRECTORY)
+    else:
+        os.symlink(source_dir, dest_dir)
+
+
+def get_bin_dir(root_dir):
+    """Locates the 'bin' directory within 'root_dir' tree."""
+    for root, dirs, _ in os.walk(root_dir):
+        if "bin" in dirs:
+            return os.path.join(root, "bin")
+    return None
+
+
+def executable_exists_in_path(executable):
+    """Returns True if 'executable' is in the PATH."""
+    return distutils.spawn.find_executable(executable) is not None
+
+
+def execute_cmd(cmd, use_file=False):
+    """Executes command and returns return_code and output from command"""
+
+    orig_cmd = ""
+    # Multi-commands need to be written to a temporary file to execute on Windows.
+    # This is due to complications with invoking Bash in Windows.
+    if use_file:
+        orig_cmd = cmd
+        with tempfile.NamedTemporaryFile(suffix=".sh", delete=False) as temp_file:
+            temp_file.write(cmd)
+        os_st = os.stat(temp_file.name)
+        os.chmod(temp_file.name, os_st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        # The temporary file name will have '\' on Windows and needs to be converted to '/'.
+        cmd = "bash -c {}".format(temp_file.name.replace("\\", "/"))
+
+    # If 'cmd' is specified as a string, convert it to a list of strings.
+    if isinstance(cmd, str):
+        cmd = shlex.split(cmd)
+
+    if use_file:
+        LOGGER.info("Executing '%s', tempfile contains: %s", cmd, orig_cmd)
+    else:
+        LOGGER.info("Executing '%s'", cmd)
+
+    try:
+        proc = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        output, _ = proc.communicate()
+        error_code = proc.returncode
+        if error_code:
+            output = "Error executing cmd {}: {}".format(cmd, output)
+    finally:
+        if use_file:
+            os.remove(temp_file.name)
+    return error_code, output
+
+
+def parse_options(options):
+    """ Parses options and returns a dict.
+
+        Since there are options which can be specifed with a short('-') or long
+        ('--') form, we preserve that in key map as {option_name: (value, form)}."""
+    options_map = collections.defaultdict(list)
+    opts = shlex.split(options)
+    for opt in opts:
+        # Handle options which could start with "-" or "--".
+        if opt.startswith("-"):
+            opt_idx = 2 if opt[1] == "-" else 1
+            opt_form = opt[:opt_idx]
+            eq_idx = opt.find("=")
+            if eq_idx == -1:
+                opt_name = opt[opt_idx:]
+                options_map[opt_name] = (None, opt_form)
+            else:
+                opt_name = opt[opt_idx:eq_idx]
+                options_map[opt_name] = (opt[eq_idx + 1:], opt_form)
+                opt_name = None
+        elif opt_name:
+            options_map[opt_name] = (opt, opt_form)
+    return options_map
+
+
+def download_file(url, file_name):
+    """Returns True if download was successful. Raises error if download fails."""
+
+    LOGGER.info("Downloading %s to %s", url, file_name)
+    with requests.Session() as session:
+        adapter = requests.adapters.HTTPAdapter(max_retries=5)
+        session.mount(url, adapter)
+        response = session.get(url, stream=True)
+        response.raise_for_status()
+
+        with open(file_name, "wb") as file_handle:
+            for block in response.iter_content(1024):
+                file_handle.write(block)
+
+        adapter.close()
+
+    return True
+
+
+def install_tarball(tarball, root_dir):
+    """ Unzip and install 'tarball' into 'root_dir'."""
+
+    LOGGER.info("Installing %s to %s", tarball, root_dir)
+    output = ""
+    extensions = [".msi", ".tgz", ".zip"]
+    ext = get_extension(tarball)
+    if ext == ".tgz":
+        with tarfile.open(tarball, "r:gz") as tar_handle:
+            tar_handle.extractall(path=root_dir)
+            output = "Unzipped {} to {}: {}".format(tarball, root_dir, tar_handle.getnames())
+        ret = 0
+    elif ext == ".zip":
+        with zipfile.ZipFile(tarball, "r") as zip_handle:
+            zip_handle.extractall(root_dir)
+            output = "Unzipped {} to {}: {}".format(tarball, root_dir, zip_handle.namelist())
+        ret = 0
+    elif ext == ".msi":
+        if not _IS_WINDOWS:
+            raise Exception("Unsupported platform for MSI install")
+        tmp_dir = tempfile.mkdtemp(dir="c:\\")
+        # Change the ownership to $USER: as ssh over Cygwin does not preserve privileges
+        # (see https://cygwin.com/ml/cygwin/2004-09/msg00087.html).
+        cmds = """
+            msiexec /a {tarball} /qn TARGETDIR="{tmp_dir}" /l msi.log ;
+            if [ $? -ne 0 ]; then
+                echo "msiexec failed to extract from {tarball}" ;
+                echo See msi.log ;
+                exit 1 ;
+            fi ;
+            mv "{tmp_dir}"/* "{root_dir}" ;
+            chown -R $USER: "{root_dir}" ;
+            chmod -R 777 "{root_dir}" ;
+            winsysdir=c:/Windows/System32 ;
+            pushd "{root_dir}/System64" ;
+            for dll in * ;
+            do
+               if [ ! -f $winsysdir/$dll ]; then
+                  echo "File $winsysdir/$dll does not exist, copying from $(pwd)" ;
+                  cp $dll $winsysdir/ ;
+               else
+                  echo "File $winsysdir/$dll already exists" ;
+               fi ;
+            done ;
+            popd ;
+            """.format(tarball=tarball, tmp_dir=tmp_dir, root_dir=root_dir)
+        ret, output = execute_cmd(cmds, use_file=True)
+        shutil.rmtree(tmp_dir)
+    else:
+        raise Exception("Unsupported file extension to unzip {},"
+                        " supported extensions are {}".format(tarball, extensions))
+
+    LOGGER.debug(output)
+    if ret:
+        raise Exception("Failed to install tarball {}, {}".format(tarball, output))
+
+
+def chmod_x_binaries(bin_dir):
+    """ Change all file permissions in 'bin_dir' to executable for everyone. """
+
+    files = os.listdir(bin_dir)
+    LOGGER.debug("chmod +x %s %s", bin_dir, files)
+    for dir_file in files:
+        bin_file = os.path.join(bin_dir, dir_file)
+        os_st = os.stat(bin_file)
+        os.chmod(bin_file, os_st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+
+def chmod_w_file(chmod_file):
+    """ Change the permission for 'chmod_file' to '+w' for everyone. """
+
+    LOGGER.debug("chmod +w %s", chmod_file)
+    if _IS_WINDOWS:
+        # The os package cannot set the directory to '+w', so we use win32security.
+        # See https://stackoverflow.com/
+        #       questions/12168110/setting-folder-permissions-in-windows-using-python
+        user, domain, sec_type = win32security.LookupAccountName("", "Everyone")
+        file_sd = win32security.GetFileSecurity(
+            chmod_file, win32security.DACL_SECURITY_INFORMATION)
+        dacl = file_sd.GetSecurityDescriptorDacl()
+        dacl.AddAccessAllowedAce(
+            win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_WRITE, user)
+        file_sd.SetSecurityDescriptorDacl(1, dacl, 0)
+        win32security.SetFileSecurity(chmod_file, win32security.DACL_SECURITY_INFORMATION, file_sd)
+    else:
+        os.chmod(chmod_file, os.stat(chmod_file) | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
+
+
+def set_windows_bootstatuspolicy():
+    """ For Windows hosts that are physical, this prevents boot to prompt after failure."""
+
+    LOGGER.info("Setting bootstatuspolicy to ignoreallfailures & boot timeout to 5 seconds")
+    cmds = """
+        echo 'Setting bootstatuspolicy to ignoreallfailures & boot timeout to 5 seconds' ;
+        bcdedit /set {default} bootstatuspolicy ignoreallfailures ;
+        bcdedit /set {current} bootstatuspolicy ignoreallfailures ;
+        bcdedit /timeout 5"""
+    ret, output = execute_cmd(cmds, use_file=True)
+    return ret, output
+
+
+def install_mongod(bin_dir=None, tarball_url="latest", root_dir=None):
+    """Sets up 'root_dir'/bin to contain MongoDB binaries.
+
+       If 'bin_dir' is specified, then symlink it to 'root_dir'/bin.
+       Otherwise, download 'tarball_url' and symlink it's bin to 'root_dir'/bin.
+
+       If 'bin_dir' is specified, skip download and create symlink
+       from 'bin_dir' to 'root_dir'/bin."""
+
+    LOGGER.debug("install_mongod: %s %s %s", bin_dir, tarball_url, root_dir)
+    # Create 'root_dir', if it does not exist.
+    root_bin_dir = os.path.join(root_dir, "bin")
+    if not os.path.isdir(root_dir):
+        LOGGER.info("install_mongod: creating %s", root_dir)
+        os.makedirs(root_dir)
+
+    # Symlink the 'bin_dir', if it's specified, to 'root_bin_dir'
+    if bin_dir and os.path.isdir(bin_dir):
+        symlink_dir(bin_dir, root_bin_dir)
+        return
+
+    if tarball_url == "latest":
+        # TODO SERVER-31021: Support all platforms.
+        if _IS_WINDOWS:
+            # MSI default:
+            # https://fastdl.mongodb.org/win32/mongodb-win32-x86_64-2008plus-ssl-latest-signed.msi
+            tarball_url = (
+                "https://fastdl.mongodb.org/win32/mongodb-win32-x86_64-2008plus-ssl-latest.zip")
+        elif _IS_LINUX:
+            tarball_url = "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-latest.tgz"
+
+    tarball = os.path.split(urlparse.urlsplit(tarball_url).path)[-1]
+    download_file(tarball_url, tarball)
+    install_tarball(tarball, root_dir)
+    chmod_x_binaries(get_bin_dir(root_dir))
+
+    # Symlink the bin dir from the tarball to 'root_bin_dir'.
+    # Since get_bin_dir returns an abolute path, we need to remove 'root_dir'
+    tarball_bin_dir = get_bin_dir(root_dir).replace("{}/".format(root_dir), "")
+    LOGGER.info("Symlink %s to %s", tarball_bin_dir, root_bin_dir)
+    symlink_dir(tarball_bin_dir, root_bin_dir)
+
+
+def print_uptime():
+    """Prints the last time the system was booted, and the uptime (in seconds). """
+    boot_time_epoch = psutil.boot_time()
+    boot_time = datetime.datetime.fromtimestamp(boot_time_epoch).strftime('%Y-%m-%d %H:%M:%S.%f')
+    uptime = int(time.time() - boot_time_epoch)
+    LOGGER.info("System was last booted %s, up %d seconds", boot_time, uptime)
+
+
+def call_remote_operation(local_ops, remote_python, script_name, client_args, operation):
+    """ Call the remote operation and returns tuple (ret, ouput). """
+    client_call = "{} {} {} {}".format(remote_python, script_name, client_args, operation)
+    ret, output = local_ops.shell(client_call)
+    return ret, output
+
+
+class ProcessControl(object):
+    """ Process control class.
+
+        Control processes either by name or a list of pids. If name is supplied, then
+        all matching pids are controlled."""
+
+    def __init__(self, name=None, pids=None):
+        """Provide either 'name' or 'pids' to control the process."""
+        if not name and not pids:
+            raise Exception("Either 'process_name' or 'pids' must be specifed")
+        self.name = name
+        self.pids = []
+        if pids:
+            self.pids = pids
+        self.procs = []
+
+    def get_pids(self):
+        """ Returns list of process ids for process 'self.name'."""
+        if not self.name:
+            return self.pids
+        self.pids = []
+        for proc in psutil.process_iter():
+            if proc.name() == self.name:
+                self.pids.append(proc.pid)
+        return self.pids
+
+    def get_name(self):
+        """ Returns process name or name of first running process from pids."""
+        if not self.name:
+            for pid in self.get_pids():
+                proc = psutil.Process(pid)
+                if psutil.pid_exists(pid):
+                    self.name = proc.name()
+                    break
+        return self.name
+
+    def get_procs(self):
+        """ Returns a list of 'proc' for the associated pids."""
+        procs = []
+        for pid in self.get_pids():
+            procs.append(psutil.Process(pid))
+        return procs
+
+    def is_running(self):
+        """ Returns true if any process is running that either matches on name or pids."""
+        for pid in self.get_pids():
+            if psutil.pid_exists(pid):
+                return True
+        return False
+
+    def terminate(self):
+        """ Terminates all running processes that match the list of pids. """
+        if self.is_running():
+            for proc in self.get_procs():
+                try:
+                    proc.terminate()
+                except psutil.NoSuchProcess:
+                    LOGGER.info("Could not terminate pid %d, process no longer exists", proc.pid)
+
+
+class WindowsService(object):
+    """ Windows service control class."""
+
+    def __init__(self,
+                 name,
+                 bin_path,
+                 bin_options,
+                 start_type=None):
+
+        self.name = name
+        self.bin_name = os.path.basename(bin_path)
+        self.bin_path = bin_path
+        self.bin_options = bin_options
+        if start_type is not None:
+            self.start_type = start_type
+        else:
+            self.start_type = win32service.SERVICE_DEMAND_START
+        self.pids = []
+        self._states = {
+            win32service.SERVICE_CONTINUE_PENDING: "continue pending",
+            win32service.SERVICE_PAUSE_PENDING: "pause pending",
+            win32service.SERVICE_PAUSED: "paused",
+            win32service.SERVICE_RUNNING: "running",
+            win32service.SERVICE_START_PENDING: "start pending",
+            win32service.SERVICE_STOPPED: "stopped",
+            win32service.SERVICE_STOP_PENDING: "stop pending",
+            }
+
+    def create(self):
+        """ Create service, if not installed. Returns (code, output) tuple. """
+        if self.status() in self._states.values():
+            return 1, "Service '{}' already installed, status: {}".format(self.name, self.status())
+        try:
+            win32serviceutil.InstallService(
+                pythonClassString="Service.{}".format(self.name),
+                serviceName=self.name,
+                displayName=self.name,
+                startType=self.start_type,
+                exeName=self.bin_path,
+                exeArgs=self.bin_options)
+            ret = 0
+            output = "Service '{}' created".format(self.name)
+        except pywintypes.error as err:
+            ret = err.winerror
+            output = "{}: {}".format(err[1], err[2])
+
+        return ret, output
+
+    def update(self):
+        """ Update installed service. Returns (code, output) tuple. """
+        if self.status() not in self._states.values():
+            return 1, "Service update '{}' status: {}".format(self.name, self.status())
+        try:
+            win32serviceutil.ChangeServiceConfig(
+                pythonClassString="Service.{}".format(self.name),
+                serviceName=self.name,
+                displayName=self.name,
+                startType=self.start_type,
+                exeName=self.bin_path,
+                exeArgs=self.bin_options)
+            ret = 0
+            output = "Service '{}' updated".format(self.name)
+        except pywintypes.error as err:
+            ret = err.winerror
+            output = "{}: {}".format(err[1], err[2])
+
+        return ret, output
+
+    def delete(self):
+        """ Delete service. Returns (code, output) tuple. """
+        if self.status() not in self._states.values():
+            return 1, "Service delete '{}' status: {}".format(self.name, self.status())
+        try:
+            win32serviceutil.RemoveService(serviceName=self.name)
+            ret = 0
+            output = "Service '{}' deleted".format(self.name)
+        except pywintypes.error as err:
+            ret = err.winerror
+            output = "{}: {}".format(err[1], err[2])
+
+        return ret, output
+
+    def start(self):
+        """ Start service. Returns (code, output) tuple. """
+        if self.status() not in self._states.values():
+            return 1, "Service start '{}' status: {}".format(self.name, self.status())
+        try:
+            win32serviceutil.StartService(serviceName=self.name)
+            ret = 0
+            output = "Service '{}' started".format(self.name)
+        except pywintypes.error as err:
+            ret = err.winerror
+            output = "{}: {}".format(err[1], err[2])
+
+        proc = ProcessControl(name=self.bin_name)
+        self.pids = proc.get_pids()
+
+        return ret, output
+
+    def stop(self):
+        """ Stop service. Returns (code, output) tuple. """
+        self.pids = []
+        if self.status() not in self._states.values():
+            return 1, "Service '{}' status: {}".format(self.name, self.status())
+        try:
+            win32serviceutil.StopService(serviceName=self.name)
+            ret = 0
+            output = "Service '{}' stopped".format(self.name)
+        except pywintypes.error as err:
+            ret = err.winerror
+            output = "{}: {}".format(err[1], err[2])
+
+        return ret, output
+
+    def status(self):
+        """ Returns state of the service as a string. """
+        try:
+            # QueryServiceStatus returns a tuple:
+            #   (scvType, svcState, svcControls, err, svcErr, svcCP, svcWH)
+            # See https://msdn.microsoft.com/en-us/library/windows/desktop/ms685996(v=vs.85).aspx
+            scv_type, svc_state, svc_controls, err, svc_err, svc_cp, svc_wh = (
+                win32serviceutil.QueryServiceStatus(serviceName=self.name))
+            if svc_state in self._states:
+                return self._states[svc_state]
+            else:
+                return "unknown"
+        except pywintypes.error:
+            return "not installed"
+
+    def get_pids(self):
+        """ Return list of pids for service. """
+        return self.pids
+
+
+class PosixService(object):
+    """ Service control on POSIX systems.
+
+        Simulates service control for background processes which fork themselves,
+        i.e., mongod with '--fork'."""
+
+    def __init__(self, name, bin_path, bin_options):
+        self.name = name
+        self.bin_path = bin_path
+        self.bin_name = os.path.basename(bin_path)
+        self.bin_options = bin_options
+        self.pids = []
+
+    def create(self):
+        """ Simulates create service. Returns (code, output) tuple. """
+        return 0, None
+
+    def update(self):
+        """ Simulates update service. Returns (code, output) tuple. """
+        return 0, None
+
+    def delete(self):
+        """ Simulates delete service. Returns (code, output) tuple. """
+        return 0, None
+
+    def start(self):
+        """ Start process. Returns (code, output) tuple. """
+        cmd = "{} {}".format(self.bin_path, self.bin_options)
+        ret, output = execute_cmd(cmd)
+        if not ret:
+            proc = ProcessControl(name=self.bin_name)
+            self.pids = proc.get_pids()
+        return ret, output
+
+    def stop(self):
+        """ Stop process. Returns (code, output) tuple. """
+        proc = ProcessControl(name=self.bin_name)
+        proc.terminate()
+        self.pids = []
+        return 0, None
+
+    def status(self):
+        """ Returns status of service. """
+        if self.get_pids():
+            return "running"
+        return "stopped"
+
+    def get_pids(self):
+        """ Return list of pids for process. """
+        return self.pids
+
+
+class MongodControl(object):
+    """ Control mongod process. """
+
+    def __init__(self, bin_dir, db_path, log_path, port, options=None):
+        extension = ".exe" if _IS_WINDOWS else ""
+        self.process_name = "mongod{}".format(extension)
+
+        self.bin_dir = bin_dir
+        if self.bin_dir:
+            self.bin_path = os.path.join(self.bin_dir, self.process_name)
+            if not os.path.isfile(self.bin_path):
+                LOGGER.error("File %s does not exist.", self.bin_path)
+        else:
+            self.bin_path = None
+
+        self.options_map = parse_options(options)
+        self.db_path = db_path
+        self.set_mongod_option("dbpath", db_path)
+        self.log_path = log_path
+        self.set_mongod_option("logpath", log_path)
+        self.set_mongod_option("logappend")
+        self.port = port
+        self.set_mongod_option("port", port)
+        self.set_mongod_option("bind_ip", "0.0.0.0")
+        if _IS_WINDOWS:
+            self.set_mongod_option("service")
+            self._service = WindowsService
+        else:
+            self.set_mongod_option("fork")
+            self._service = PosixService
+        # After mongod has been installed, self.bin_path is defined.
+        if self.bin_path:
+            self.service = self._service("mongod-powertest", self.bin_path, self.mongod_options())
+
+    def set_mongod_option(self, option, option_value=None, option_form="--"):
+        """ Sets mongod command line option. """
+        self.options_map[option] = (option_value, option_form)
+
+    def get_mongod_option(self, option):
+        """ Returns tuple of (value, form). """
+        return self.options_map[option]
+
+    def get_mongod_service(self):
+        """ Returns the service object used to control mongod. """
+        return self.service
+
+    def mongod_options(self):
+        """ Returns string of mongod options, which can be used when invoking mongod. """
+        opt_string = ""
+        for opt_name in self.options_map:
+            opt_val, opt_form = self.options_map[opt_name]
+            opt_string += " {}{}".format(opt_form, opt_name)
+            if opt_val:
+                opt_string += " {}".format(opt_val)
+        return opt_string
+
+    def install(self, root_dir, tarball_url):
+        """ Returns tuple (ret, ouput). """
+        # Install mongod, if 'root_dir' does not exist.
+        if os.path.isdir(root_dir):
+            LOGGER.warning("Root dir %s already exists", root_dir)
+        else:
+            install_mongod(bin_dir=self.bin_dir, tarball_url=tarball_url, root_dir=root_dir)
+        self.bin_dir = get_bin_dir(root_dir)
+        if not self.bin_dir:
+            ret, output = execute_cmd("ls -lR '{}'".format(root_dir), use_file=True)
+            LOGGER.debug(output)
+            return 1, "No bin dir can be found under {}".format(root_dir)
+        self.bin_path = os.path.join(self.bin_dir, self.process_name)
+        # We need to instantiate the Service when installing, since the bin_path
+        # is only known after install_mongod runs.
+        self.service = self._service("mongod-powertest", self.bin_path, self.mongod_options())
+        ret, output = self.service.create()
+        return ret, output
+
+    def uninstall(self):
+        """ Returns tuple (ret, ouput). """
+        return self.service.delete()
+
+    def cleanup(self, root_dir):
+        """ Returns tuple (ret, ouput). """
+        shutil.rmtree(root_dir, ignore_errors=True)
+        return 0, None
+
+    def start(self):
+        """ Returns tuple (ret, ouput). """
+        return self.service.start()
+
+    def update(self):
+        """ Returns tuple (ret, ouput). """
+        return self.service.update()
+
+    def stop(self):
+        """ Returns tuple (ret, ouput). """
+        return self.service.stop()
+
+    def get_pids(self):
+        """ Return list of pids for process. """
+        return self.service.get_pids()
+
+
+class LocalToRemoteOperations(object):
+    """ Local operations handler class for sending commands to the remote host.
+
+        Returns (return code, output). """
+
+    def __init__(self,
+                 user_host,
+                 ssh_connection_options=None,
+                 ssh_options=None,
+                 shell_binary="/bin/bash",
+                 use_shell=False):
+
+        self.remote_op = remote_operations.RemoteOperations(
+            user_host=user_host,
+            ssh_connection_options=ssh_connection_options,
+            ssh_options=ssh_options,
+            retries=10,
+            retry_sleep=10,
+            debug=True,
+            shell_binary=shell_binary,
+            use_shell=use_shell)
+
+    def shell(self, cmds, remote_dir=None):
+        """ Returns tuple (ret, output) from performing remote shell operation. """
+        return self.remote_op.shell(cmds, remote_dir)
+
+    def copy_from(self, files, remote_dir=None):
+        """ Returns tuple (ret, output) from performing remote copy_to operation. """
+        return self.remote_op.copy_from(files, remote_dir)
+
+    def copy_to(self, files, remote_dir=None):
+        """ Returns tuple (ret, output) from performing remote copy_from operation. """
+        return self.remote_op.copy_to(files, remote_dir)
+
+
+def remote_handler(options, operations):
+    """ Remote operations handler executes all remote operations on the remote host.
+
+        These operations are invoked on the remote host's copy of this script.
+        Only one operation can be performed at a time. """
+
+    # Set 'root_dir' to absolute path.
+    root_dir = abs_path(options.root_dir)
+    if not operations:
+        raise ValueError("No remote operation specified.")
+
+    print_uptime()
+    LOGGER.info("Operations to perform %s", operations)
+    host_port = "localhost:{}".format(options.port)
+
+    if options.use_replica_set and options.repl_set:
+        options.mongod_options = "{} --replSet {}".format(
+            options.mongod_options, options.repl_set)
+
+    # For MongodControl, the file references should be fully specified.
+    if options.mongodb_bin_dir:
+        bin_dir = abs_path(options.mongodb_bin_dir)
+    else:
+        bin_dir = get_bin_dir(root_dir)
+    db_path = abs_path(options.db_path)
+    log_path = abs_path(options.log_path)
+
+    mongod = MongodControl(
+        bin_dir=bin_dir,
+        db_path=db_path,
+        log_path=log_path,
+        port=options.port,
+        options=options.mongod_options)
+
+    # Perform the sequence of operations specified. If any operation fails
+    # then return immediately.
+    for operation in operations:
+        # This is the internal "crash" mechanism, which is executed on the remote host.
+        if operation == "crash_server":
+            ret, output = internal_crash(options.remote_sudo)
+
+        elif operation == "install_mongod":
+            ret, output = mongod.install(root_dir, options.tarball_url)
+            LOGGER.info(output)
+
+            # Create mongod's dbpath, if it does not exist.
+            if not os.path.isdir(db_path):
+                os.makedirs(db_path)
+
+            # Create mongod's logpath directory, if it does not exist.
+            log_dir = os.path.dirname(log_path)
+            if not os.path.isdir(log_dir):
+                os.makedirs(log_dir)
+
+            # Windows special handling.
+            if _IS_WINDOWS:
+                # The os package cannot set the directory to '+w'
+                # See https://docs.python.org/2/library/os.html#os.chmod
+                chmod_w_file(db_path)
+                chmod_w_file(log_dir)
+                # Disable boot prompt after system crash.
+                ret, output = set_windows_bootstatuspolicy()
+                LOGGER.info(output)
+
+        elif operation == "start_mongod":
+            # Always update the service before starting, as options might have changed.
+            ret, output = mongod.update()
+            LOGGER.info(output)
+            ret, output = mongod.start()
+            LOGGER.info(output)
+            if ret:
+                LOGGER.error("Failed to start mongod on port %d: %s", options.port, output)
+                return ret
+            LOGGER.info("Started mongod running on port %d pid %s",
+                        options.port, mongod.get_pids())
+            mongo = pymongo.MongoClient(host="localhost", port=options.port)
+            LOGGER.info("Server buildinfo: %s", mongo.admin.command("buildinfo"))
+            LOGGER.info("Server serverStatus: %s", mongo.admin.command("serverStatus"))
+            if options.use_replica_set and options.repl_set:
+                ret = mongo_reconfig_replication(mongo, host_port, options.repl_set)
+            ret = 0 if not ret else 1
+
+        elif operation == "stop_mongod":
+            ret, output = mongod.stop()
+            LOGGER.info(output)
+            ret = wait_for_mongod_shutdown(options.db_path)
+
+        elif operation == "shutdown_mongod":
+            mongo = pymongo.MongoClient(host="localhost", port=options.port)
+            try:
+                mongo.admin.command("shutdown", force=True)
+            except pymongo.errors.AutoReconnect:
+                pass
+            ret = wait_for_mongod_shutdown(options.db_path)
+
+        elif operation == "rsync_data":
+            ret, output = rsync(options.db_path, options.rsync_dest)
+            LOGGER.info(output)
+
+        elif operation == "seed_docs":
+            mongo = pymongo.MongoClient(host="localhost", port=options.port)
+            ret = mongo_seed_docs(
+                mongo, options.db_name, options.collection_name, options.seed_doc_num)
+
+        elif operation == "validate_collections":
+            mongo = pymongo.MongoClient(host="localhost", port=options.port)
+            ret = mongo_validate_collections(mongo)
+
+        elif operation == "insert_canary":
+            mongo = pymongo.MongoClient(host="localhost", port=options.port)
+            ret = mongo_insert_canary(
+                mongo, options.db_name, options.collection_name, options.canary_doc)
+
+        elif operation == "validate_canary":
+            mongo = pymongo.MongoClient(host="localhost", port=options.port)
+            ret = mongo_validate_canary(
+                mongo, options.db_name, options.collection_name, options.canary_doc)
+
+        else:
+            LOGGER.error("Unsupported remote option specified '%s'", operation)
+            ret = 1
+
+        if ret:
+            return ret
+
+    return 0
+
+
+def rsync(src_dir, dest_dir):
+    """ Rsync 'src_dir' to 'dest_dir'. """
+    # Note rsync on Windows requires a Unix-style directory.
+    LOGGER.info("Rsync'ing %s to %s", src_dir, dest_dir)
+    if not executable_exists_in_path("rsync"):
+        return 1, "No rsync exists on the host, not rsync'ing"
+    cmds = "rsync -va --delete --quiet {} {}".format(src_dir, dest_dir)
+    ret, output = execute_cmd(cmds)
+    return ret, output
+
+
+def internal_crash(use_sudo=False):
+    """ Internally crash the host this excutes on. """
+
+    # Windows does not have a way to immediately crash itself. It's
+    # better to use an external mechanism instead.
+    if _IS_WINDOWS:
+        # Sleep after issuing shutdown, to prevent the 'client' side script
+        # continuing, as shutdown is no immediate.
+        cmds = """
+            shutdown /r /f /t 0 ;
+            sleep 10"""
+        ret, output = execute_cmd(cmds, use_file=True)
+        return ret, output
+    else:
+        # These operations simulate a console boot and require root privileges, see:
+        # - http://www.linuxjournal.com/content/rebooting-magic-way
+        # - https://www.mjmwired.net/kernel/Documentation/sysrq.txt
+        # These file operations could be performed natively,
+        # however since they require root (or sudo), we prefer to do them
+        # in a subprocess call to isolate them and not require the invocation
+        # of this script to be with sudo.
+        # Code to perform natively:
+        #   with open("/proc/sys/kernel/sysrq", "w") as f:
+        #       f.write("1\n")
+        #   with open("/proc/sysrq-trigger", "w") as f:
+        #       f.write("b\n")
+        sudo = "/usr/bin/sudo" if use_sudo else ""
+        cmds = """
+            echo "Server crashing now" | {sudo} wall ;
+            echo 1 | {sudo} tee /proc/sys/kernel/sysrq ;
+            echo b | {sudo} tee /proc/sysrq-trigger""".format(sudo=sudo)
+        ret, output = execute_cmd(cmds, use_file=True)
+    LOGGER.debug(output)
+    return 1, "Crash did not occur"
+
+
+def crash_server(options, crash_canary, local_ops, script_name, client_args):
+    """ Crashes server and optionally writes canary doc before crash. """
+
+    crash_wait_time = options.crash_wait_time + random.randint(0, options.crash_wait_time_jitter)
+    LOGGER.info("Crashing server in %d seconds", crash_wait_time)
+    time.sleep(crash_wait_time)
+
+    crash_func = local_ops.shell
+
+    if options.crash_method == "mpower":
+        # Provide time for power to dissipate by sleeping 10 seconds before turning it back on.
+        crash_args = ["""
+            echo 0 > /dev/{crash_options} ;
+            sleep 10 ;
+            echo 1 > /dev/{crash_options}""".format(crash_options=options.crash_options)]
+        local_ops = LocalToRemoteOperations(
+            user_host=options.ssh_crash_user_host,
+            ssh_connection_options=options.ssh_crash_options,
+            shell_binary="/bin/sh")
+        crash_func = local_ops.shell
+
+    elif options.crash_method == "internal":
+        if options.canary == "remote":
+            # The crash canary function executes remotely, only if the
+            # crash_method is 'internal'.
+            canary = "--docForCanary \"{}\"".format(crash_canary["args"][3])
+            canary_cmd = "insert_canary"
+        else:
+            canary = ""
+            canary_cmd = ""
+        crash_args = ["{} {} --remoteOperation {} {} {} crash_server".format(
+            options.remote_python,
+            script_name,
+            client_args,
+            canary,
+            canary_cmd)]
+
+    elif options.crash_method == "aws_ec2":
+        ec2 = aws_ec2.AwsEc2()
+        crash_func = ec2.control_instance
+        crash_args = ["force-stop", options.crash_options]
+
+    else:
+        LOGGER.error("Unsupported crash method '%s' provided", options.crash_method)
+        return False
+
+    # Invoke the crash canary function, right before crashing the server.
+    if crash_canary and options.canary == "local":
+        crash_canary["function"](*crash_canary["args"])
+    _, output = crash_func(*crash_args)
+    LOGGER.info(output)
+
+
+def wait_for_mongod_shutdown(data_dir, timeout=120):
+    """ Waits for for mongod to shutdown.
+
+        Returns 0 if shutdown occurs within 'timeout', else 1. """
+
+    lock_file = os.path.join(data_dir, "mongod.lock")
+    LOGGER.info("Waiting for mongod to release lockfile %s", lock_file)
+    start = time.time()
+    while os.path.exists(lock_file) and os.stat(lock_file).st_size:
+        time.sleep(3)
+        if time.time() - start >= timeout:
+            LOGGER.error("The mongod lockfile %s has not been released, exiting", lock_file)
+            return 1
+    LOGGER.info("The mongod lockfile %s has been released", lock_file)
+    return 0
+
+
+def get_mongo_client_args(options):
+    """ Returns keyword arg dict used in PyMongo client. """
+    mongo_args = {}
+    # Set the writeConcern
+    mongo_args = options.write_concern
+    # Set the readConcernLevel
+    if options.read_concern_level:
+        mongo_args["readConcernLevel"] = options.read_concern_level
+    return mongo_args
+
+
+def mongo_shell(mongo_path, work_dir, host_port, mongo_cmds, retries=5, retry_sleep=5):
+    """Starts mongo_path from work_dir, connecting to host_port and executes mongo_cmds."""
+    cmds = """
+            cd {};
+            echo '{}' | {} {}""".format(work_dir, mongo_cmds, mongo_path, host_port)
+    attempt_num = 0
+    while True:
+        ret, output = execute_cmd(cmds, use_file=True)
+        if not ret:
+            break
+        attempt_num += 1
+        if attempt_num > retries:
+            break
+        time.sleep(retry_sleep)
+    return ret, output
+
+
+def mongod_wait_for_primary(mongo, timeout=60, sleep_interval=3):
+    """ Return True if the mongod primary is available in replica set,
+        within the specified timeout."""
+
+    start = time.time()
+    while not mongo.admin.command("isMaster")["ismaster"]:
+        time.sleep(sleep_interval)
+        if time.time() - start >= timeout:
+            return False
+    return True
+
+
+def mongo_reconfig_replication(mongo, host_port, repl_set):
+    """ Reconfigure the mongod replica set. Return 0 if successful."""
+
+    # TODO: Rework reconfig logic as follows:
+    # 1. Start up mongod in standalone
+    # 2. Delete the config doc
+    # 3. Stop mongod
+    # 4. Start mongod
+    # When reconfiguring the replica set, due to a switch in ports
+    # it can only be done using force=True, as the node will not come up as Primary.
+    # The side affect of using force=True are large jumps in the config
+    # version, which after many reconfigs may exceed the 'int' value.
+
+    database = pymongo.database.Database(mongo, "local")
+    system_replset = database.get_collection("system.replset")
+    # Check if replica set has already been initialized
+    if not system_replset or not system_replset.find_one():
+        rs_config = {"_id": repl_set, "members": [{"_id": 0, "host": host_port}]}
+        ret = mongo.admin.command("replSetInitiate", rs_config)
+    else:
+        ret = mongo.admin.command("replSetGetConfig")
+        if ret["ok"] != 1:
+            return 1
+        rs_config = ret["config"]
+        # We only reconfig if there is a change to 'host'.
+        if rs_config["members"][0]["host"] != host_port:
+            # With force=True, version is ignored.
+            # rs_config["version"] = rs_config["version"] + 1
+            rs_config["members"][0]["host"] = host_port
+            ret = mongo.admin.command("replSetReconfig", rs_config, force=True)
+    primary_available = mongod_wait_for_primary(mongo)
+    LOGGER.debug("isMaster: %s", mongo.admin.command("isMaster"))
+    LOGGER.debug("replSetGetStatus: %s", mongo.admin.command("replSetGetStatus"))
+    return 0 if ret["ok"] == 1 and primary_available else 1
+
+
+def mongo_seed_docs(mongo, db_name, coll_name, num_docs):
+    """ Seed a collection with random document values. """
+
+    def rand_string(max_length=1024):
+        """Returns random string of random length. """
+        return ''.join(random.choice(string.letters) for _ in range(random.randint(1, max_length)))
+
+    LOGGER.info("Seeding DB '%s' collection '%s' with %d documents, %d already exist",
+                db_name, coll_name, num_docs, mongo[db_name][coll_name].count())
+    random.seed()
+    base_num = 100000
+    bulk_num = min(num_docs, 10000)
+    bulk_loops = num_docs / bulk_num
+    for _ in xrange(bulk_loops):
+        num_coll_docs = mongo[db_name][coll_name].count()
+        if num_coll_docs >= num_docs:
+            break
+        mongo[db_name][coll_name].insert_many(
+            [{"x": random.randint(0, base_num), "doc": rand_string(1024)}
+             for _ in xrange(bulk_num)])
+    LOGGER.info("After seeding there are %d documents in the collection",
+                mongo[db_name][coll_name].count())
+    return 0
+
+
+def mongo_validate_collections(mongo):
+    """ Validates the mongo collections. Returns 0 if all are valid. """
+
+    LOGGER.info("Validating all collections")
+    invalid_colls = []
+    ebusy_colls = []
+    for db_name in mongo.database_names():
+        for coll_name in mongo[db_name].collection_names():
+            res = mongo[db_name].command({"validate": coll_name, "full": True})
+            LOGGER.info("Validating %s %s: %s", db_name, coll_name, res)
+            ebusy = "EBUSY" in res["errors"] or "EBUSY" in res["warnings"]
+            if not res["valid"]:
+                invalid_colls.append(coll_name)
+            elif ebusy:
+                ebusy_colls.append(coll_name)
+    if ebusy_colls:
+        LOGGER.warning("EBUSY collections: %s", ebusy_colls)
+    if invalid_colls:
+        LOGGER.error("Invalid collections: %s", ebusy_colls)
+
+    return 0 if not invalid_colls else 1
+
+
+def mongo_validate_canary(mongo, db_name, coll_name, doc):
+    """ Validates a canary document. Returns 0 if the document exists. """
+    LOGGER.info("Validating canary document %s", doc)
+    return 0 if not doc or mongo[db_name][coll_name].find_one(doc) else 1
+
+
+def mongo_insert_canary(mongo, db_name, coll_name, doc):
+    """ Inserts a canary document with 'j' True. Returns 0 if successful. """
+    LOGGER.info("Inserting canary document %s", doc)
+    coll = mongo[db_name][coll_name].with_options(
+        write_concern=pymongo.write_concern.WriteConcern(j=True))
+    res = coll.insert_one(doc)
+    return 0 if res.inserted_id else 1
+
+
+def main():
+    """ Main program. """
+
+    parser = optparse.OptionParser(usage="""
+%prog [options]
+
+MongoDB Powercycle test
+
+Examples:
+
+    Server is running as single node replica set connected to mFi mPower, outlet1:
+      python powertest.py
+            --sshUserHost 10.4.1.54
+            --rootDir pt-mmap
+            --replSet power
+            --crashMethod mpower
+            --crashOptions output1
+            --sshCrashUserHost admin@10.4.100.2
+            --sshCrashOptions "-oKexAlgorithms=+diffie-hellman-group1-sha1 -i /Users/jonathan/.ssh/mFi.pem"
+            --mongodOptions "--storageEngine mmapv1"
+
+    Linux server running in AWS, testing nojournal:
+      python powertest.py
+            --sshUserHost ec2-user@52.4.173.196
+            --sshConnection "-i $HOME/.ssh/JAkey.pem"
+            --rootDir pt-nojournal
+            --mongodOptions "--nojournal"
+""")
+
+    test_options = optparse.OptionGroup(parser, "Test Options")
+    crash_options = optparse.OptionGroup(parser, "Crash Options")
+    mongodb_options = optparse.OptionGroup(parser, "MongoDB Options")
+    mongod_options = optparse.OptionGroup(parser, "mongod Options")
+    program_options = optparse.OptionGroup(parser, "Program Options")
+
+    # Test options
+    test_options.add_option("--sshUserHost",
+                            dest="ssh_user_host",
+                            help="Server ssh user/host, i.e., user@host (REQUIRED)",
+                            default=None)
+
+    default_ssh_connection_options = (
+        "-o ServerAliveCountMax=10"
+        " -o ServerAliveInterval=6"
+        " -o StrictHostKeyChecking=no"
+        " -o ConnectTimeout=30"
+        " -o ConnectionAttempts=25")
+    test_options.add_option("--sshConnection",
+                            dest="ssh_connection_options",
+                            help="Server ssh additional connection options, i.e., '-i ident.pem'"
+                                 " which are added to '{}'".format(default_ssh_connection_options),
+                            default=None)
+
+    test_options.add_option("--mongoPath",
+                            dest="mongo_path",
+                            help="Path to mongo (shell) executable, if unspecifed, mongo client"
+                                 " is launched from $PATH",
+                            default="mongo")
+
+    test_options.add_option("--mongoRepoRootDir",
+                            dest="mongo_repo_root_dir",
+                            help="Root directory of mongoDB repository, defaults to current"
+                                 " directory.",
+                            default=None)
+
+    test_options.add_option("--testLoops",
+                            dest="num_loops",
+                            help="Number of powercycle loops to run [default: %default]",
+                            type="int",
+                            default=10)
+
+    test_options.add_option("--testTime",
+                            dest="test_time",
+                            help="Time to run test (in seconds), overrides --testLoops",
+                            type="int",
+                            default=0)
+
+    test_options.add_option("--rsync",
+                            dest="rsync_data",
+                            help="Rsync data directory between mongod stop and start",
+                            action="store_true",
+                            default=False)
+
+    validate_locations = ["local", "remote"]
+    test_options.add_option("--validate",
+                            dest="validate_collections",
+                            help="Run validate on all collections after mongod restart after"
+                                 " a powercycle. Choose from {} to specify where the"
+                                 " validate runs.".format(validate_locations),
+                            choices=validate_locations,
+                            default=None)
+
+    canary_locations = ["local", "remote"]
+    test_options.add_option("--canary",
+                            dest="canary",
+                            help="Generate and validate canary document between powercycle"
+                                 " events. Choose from {} to specify where the canary is"
+                                 " generated from. If the 'crashMethod' is not 'internal"
+                                 " then this option must be 'local'.".format(canary_locations),
+                            choices=canary_locations,
+                            default=None)
+
+    test_options.add_option("--docForCanary",
+                            dest="canary_doc",
+                            help=optparse.SUPPRESS_HELP,
+                            default="")
+
+    test_options.add_option("--seedDocNum",
+                            dest="seed_doc_num",
+                            help="Number of documents to seed the default collection [default:"
+                                 " %default]",
+                            type="int",
+                            default=0)
+
+    test_options.add_option("--dbName",
+                            dest="db_name",
+                            help=optparse.SUPPRESS_HELP,
+                            default="power")
+
+    test_options.add_option("--collectionName",
+                            dest="collection_name",
+                            help=optparse.SUPPRESS_HELP,
+                            default="cycle")
+
+    test_options.add_option("--writeConcern",
+                            dest="write_concern",
+                            help="mongo (shell) CRUD client writeConcern, i.e.,"
+                                 " '{\"w\": \"majority\"}' [default: '%default']",
+                            default="{}")
+
+    test_options.add_option("--readConcernLevel",
+                            dest="read_concern_level",
+                            help="mongo (shell) CRUD client readConcernLevel, i.e.,"
+                                 "'majority'",
+                            default=None)
+
+    # Crash options
+    crash_methods = ["aws_ec2", "internal", "mpower"]
+    crash_options.add_option("--crashMethod",
+                             dest="crash_method",
+                             choices=crash_methods,
+                             help="Crash methods: {} [default: '%default']".format(crash_methods),
+                             default="internal")
+
+    crash_options.add_option("--crashWaitTime",
+                             dest="crash_wait_time",
+                             help="Time, in seconds, to wait before issuing crash [default:"
+                                  " %default]",
+                             type="int",
+                             default=30)
+
+    crash_options.add_option("--jitterForCrashWaitTime",
+                             dest="crash_wait_time_jitter",
+                             help="The maximum time, in seconds, to be added to --crashWaitTime,"
+                                  " as a uniform distributed random value,"
+                                  " [default: %default]",
+                             type="int",
+                             default=10)
+
+    crash_options.add_option("--sshCrashUserHost",
+                             dest="ssh_crash_user_host",
+                             help="The crash host's user@host for performing the crash.",
+                             default=None)
+
+    crash_options.add_option("--sshCrashOptions",
+                             dest="ssh_crash_options",
+                             help="The crash host's ssh connection options, i.e., '-i ident.pem'",
+                             default=None)
+
+    crash_options.add_option("--crashOptions",
+                             dest="crash_options",
+                             help="Secondary argument for the following --crashMethod:"
+                                  " 'aws_ec2': specify EC2 instance_id."
+                                  " 'mpower': specify output<num> to turn off/on, i.e.,"
+                                  " 'output1'.",
+                             default=None)
+
+    # MongoDB options
+    mongodb_options.add_option("--downloadUrl",
+                               dest="tarball_url",
+                               help="URL of tarball to test, if unspecifed latest tarball will be"
+                                    " used",
+                               default="latest")
+
+    mongodb_options.add_option("--rootDir",
+                               dest="root_dir",
+                               help="Root directory, on remote host, to install tarball and data"
+                                    " directory [default: 'mongodb-powertest-<epochSecs>']",
+                               default=None)
+
+    mongodb_options.add_option("--mongodbBinDir",
+                               dest="mongodb_bin_dir",
+                               help="Directory, on remote host, containing mongoDB binaries,"
+                                    " overrides bin from tarball in --downloadUrl",
+                               default=None)
+
+    mongodb_options.add_option("--dbPath",
+                               dest="db_path",
+                               help="Data directory to use, on remote host, if unspecified"
+                                    " it will be '<rootDir>/data/db'",
+                               default=None)
+
+    mongodb_options.add_option("--logPath",
+                               dest="log_path",
+                               help="Log path, on remote host, if unspecified"
+                                    " it will be '<rootDir>/log/mongod.log'",
+                               default=None)
+
+    # mongod options
+    mongod_options.add_option("--replSet",
+                              dest="repl_set",
+                              help="Name of mongod single node replica set, if unpsecified mongod"
+                                   " defaults to standalone node",
+                              default=None)
+
+    # The current port used to start and connect to mongod. Not meant to be specified
+    # by the user.
+    mongod_options.add_option("--mongodPort",
+                              dest="port",
+                              help=optparse.SUPPRESS_HELP,
+                              type="int",
+                              default=None)
+
+    mongod_options.add_option("--useReplicaSet",
+                              dest="use_replica_set",
+                              help=optparse.SUPPRESS_HELP,
+                              action="store_true",
+                              default=False)
+
+    # The ports used on the 'server' side when in standard or secret mode.
+    mongod_options.add_option("--mongodUsablePorts",
+                              dest="usable_ports",
+                              nargs=2,
+                              help="List of usable ports to be used by mongod for"
+                                   " standard and secret modes, [default: %default]",
+                              type="int",
+                              default=[27017, 37017])
+
+    mongod_options.add_option("--mongodOptions",
+                              dest="mongod_options",
+                              help="Additional mongod options",
+                              default="")
+
+    # Program options
+    program_options.add_option("--remotePython",
+                               dest="remote_python",
+                               help="The python intepreter to use on the remote host"
+                                    " [default: '%default']."
+                                    " To be able to use a python virtual environment,"
+                                    " which has already been provisioned on the remote"
+                                    " host, specify something similar to this:"
+                                    " 'source venv/bin/activate;  python'",
+                               default="python")
+
+    # Program options
+    program_options.add_option("--remoteSudo",
+                               dest="remote_sudo",
+                               help="Use sudo on the remote host for priveleged operations."
+                                    " [default: %default]."
+                                    " For non-Windows systems, in order to perform privileged"
+                                    " operations on the remote host, specify this, if the"
+                                    " remote user is not able to perform root operations.",
+                               action="store_true",
+                               default=False)
+
+    log_levels = ["debug", "info", "warning", "error"]
+    program_options.add_option("--logLevel",
+                               dest="log_level",
+                               choices=log_levels,
+                               help="The log level. Accepted values are: {}."
+                                    " [default: '%default'].".format(log_levels),
+                               default="info")
+
+    program_options.add_option("--logFile",
+                               dest="log_file",
+                               help="The destination file for the log output. Defaults to stdout.",
+                               default=None)
+
+    program_options.add_option("--version",
+                               dest="version",
+                               help="Display this program's version",
+                               action="store_true",
+                               default=False)
+
+    # Remote options, include commands and options sent from client to server under test.
+    # These are 'internal' options, not meant to be directly specifed.
+    # More than one remote operation can be provided and they are specified in the program args.
+    program_options.add_option("--remoteOperation",
+                               dest="remote_operation",
+                               help=optparse.SUPPRESS_HELP,
+                               action="store_true",
+                               default=False)
+
+    program_options.add_option("--backupPathBefore",
+                               dest="backup_path_before",
+                               help="Path where the db_path is backed up before crash recovery,"
+                                    " defaults to '<rootDir>/data-beforerecovery/db'",
+                               default=None)
+
+    program_options.add_option("--backupPathAfter",
+                               dest="backup_path_after",
+                               help="Path where the db_path is backed up after crash recovery,"
+                                    " defaults to '<rootDir>/data-afterrecovery/db'",
+                               default=None)
+
+    program_options.add_option("--rsyncDest",
+                               dest="rsync_dest",
+                               help=optparse.SUPPRESS_HELP,
+                               default=None)
+
+    parser.add_option_group(test_options)
+    parser.add_option_group(crash_options)
+    parser.add_option_group(mongodb_options)
+    parser.add_option_group(mongod_options)
+    parser.add_option_group(program_options)
+
+    options, args = parser.parse_args()
+
+    logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s",
+                        level=options.log_level.upper(), filename=options.log_file)
+    logging.Formatter.converter = time.gmtime
+
+    LOGGER.info("powertest.py invocation: %s", " ".join(sys.argv))
+
+    script_name = os.path.basename(__file__)
+    # Print script name and version.
+    if options.version:
+        print("{}:{}".format(script_name, __version__))
+        sys.exit(0)
+
+    # Initialize the mongod options
+    if not options.root_dir:
+        options.root_dir = "mongodb-powertest-{}".format(int(time.time()))
+    if not options.db_path:
+        options.db_path = os.path.join(options.root_dir, "data", "db")
+    if not options.log_path:
+        options.log_path = os.path.join(options.root_dir, "log", "mongod.log")
+    mongod_options_map = parse_options(options.mongod_options)
+
+    # Error out earlier if these options are not properly specified
+    options.write_concern = yaml.safe_load(options.write_concern)
+    options.canary_doc = yaml.safe_load(options.canary_doc)
+
+    # Invoke remote_handler if remote_operation is specified.
+    # The remote commands are program args.
+    if options.remote_operation:
+        ret = remote_handler(options, args)
+        # Exit here since the local operations are performed after this.
+        sys.exit(ret)
+
+    # Required option for non-remote commands.
+    if options.ssh_user_host is None and not options.remote_operation:
+        parser.error("Missing required argument --sshUserHost")
+
+    secret_port = options.usable_ports[1]
+    standard_port = options.usable_ports[0]
+
+    seed_docs = "seed_docs" if options.seed_doc_num else ""
+
+    if options.rsync_data:
+        rsync_cmd = "rsync_data"
+    else:
+        rsync_cmd = ""
+        rsync_opt = ""
+
+    # Setup the mongo_repo_root
+    mongo_repo_root_dir = "." if not options.mongo_repo_root_dir else options.mongo_repo_root_dir
+    if not os.path.isdir(mongo_repo_root_dir):
+        LOGGER.error("mongoRepoRoot %s does not exist", mongo_repo_root_dir)
+        sys.exit(1)
+
+    # Setup the validate_collections option
+    if options.validate_collections == "remote":
+        validate_collections_cmd = "validate_collections"
+    else:
+        validate_collections_cmd = ""
+
+    # Setup the validate_canary option
+    if options.canary and "nojournal" in mongod_options_map:
+        LOGGER.error("Cannot create and validate canary documents if the mongod option"
+                     " '--nojournal' is used.")
+        sys.exit(1)
+    if options.canary == "remote" and options.crash_method != "internal":
+        parser.error("The option --canary can only be specified as 'remote' if --crashMethod"
+                     " is 'internal'")
+    orig_canary_doc = canary_doc = ""
+    validate_canary_cmd = ""
+
+    # The remote mongod host comes from the ssh_user_host,
+    # which may be specified as user@host.
+    mongod_host = options.ssh_user_host.rsplit()[-1].rsplit("@")[-1]
+
+    ssh_connection_options = "{} {}".format(
+        default_ssh_connection_options,
+        options.ssh_connection_options if options.ssh_connection_options else "")
+    # For remote operations requiring sudo, force pseudo-tty allocation,
+    # see https://stackoverflow.com/questions/10310299/proper-way-to-sudo-over-ssh.
+    # Note - the ssh option RequestTTY was added in OpenSSH 5.9, so we use '-tt'.
+    ssh_options = "-tt" if options.remote_sudo else None
+
+    # Instantiate the local handler object.
+    local_ops = LocalToRemoteOperations(
+        user_host=options.ssh_user_host,
+        ssh_connection_options=ssh_connection_options,
+        ssh_options=ssh_options,
+        use_shell=True)
+
+    # Bootstrap the remote host with this script.
+    ret, output = local_ops.copy_to(__file__)
+    if ret:
+        LOGGER.error("Cannot access remote system %s", output)
+        sys.exit(1)
+
+    # Pass client_args to the remote script invocation.
+    client_args = ""
+    for option in parser._get_all_options():
+        if option.dest:
+            option_value = getattr(options, option.dest)
+            if option_value != option.default:
+                # The boolean options do not require the option_value.
+                if isinstance(option_value, bool):
+                    option_value = ""
+                # Quote the non-default option values from the invocation of this script,
+                # if they have spaces, or quotes, such that they can be safely passed to the
+                # remote host's invocation of this script.
+                elif isinstance(option_value, str) and re.search("\"|'| ", option_value):
+                    option_value = "'{}'".format(option_value)
+                # The tuple options need to be changed to a string.
+                elif isinstance(option_value, tuple):
+                    option_value = " ".join(map(str, option_value))
+                client_args = "{} {} {}".format(client_args, option.get_opt_string(), option_value)
+
+    LOGGER.info("%s %s", __file__, client_args)
+
+    # Remote install of MongoDB.
+    ret, output = call_remote_operation(
+        local_ops,
+        options.remote_python,
+        script_name,
+        client_args,
+        "--remoteOperation install_mongod")
+    LOGGER.info("****install_mongod: %d %s****", ret, output)
+    if ret:
+        sys.exit(ret)
+
+    # test_time option overrides num_loops.
+    if options.test_time:
+        options.num_loops = 999999
+    else:
+        options.test_time = 999999
+    loop_num = 0
+    start_time = int(time.time())
+    test_time = 0
+
+    # ======== Main loop for running the powercycle test========:
+    #   1. Rsync the database (optional, post-crash, pre-recovery)
+    #   2. Start mongod on the secret port and wait for it to recover
+    #   3  Validate collections (optional)
+    #   4. Validate canary (optional)
+    #   5. Stop mongod
+    #   6. Rsync the database (optional, post-recovery)
+    #   7. Start mongod on the standard port
+    #   8. Start mongo (shell) & FSM clients
+    #   9. Generate canary document (optional)
+    #  10. Crash the server
+    #  11. Exit loop if one of these occurs:
+    #      a. Loop time or loop number exceeded
+    #      b. Any step fails
+    # =========
+    while True:
+        loop_num += 1
+        LOGGER.info("****Starting test loop %d test time %d seconds****", loop_num, test_time)
+
+        if options.canary and loop_num > 1:
+            canary_opt = "--docForCanary \"{}\"".format(canary_doc)
+            validate_canary_cmd = "validate_canary" if options.canary else ""
+        else:
+            canary_opt = ""
+
+        # Since rsync requires Posix style paths, we do not use os.path.join to
+        # construct the rsync destination directory.
+        if rsync_cmd:
+            if options.backup_path_before:
+                rsync_dest = options.backup_path_before
+            else:
+                rsync_dest = "{}/data-afterrecovery".format(options.root_dir)
+            rsync_opt = " --rsyncDest {}".format(rsync_dest)
+
+        # Optionally, rsync the pre-recovery database.
+        # Start monogd on the secret port.
+        # Optionally validate collections, validate the canary and seed the collection.
+        remote_operation = ("--remoteOperation"
+                            " {rsync_opt}"
+                            " {canary_opt}"
+                            " --mongodPort {port}"
+                            " {rsync_cmd}"
+                            " start_mongod"
+                            " {validate_collections_cmd}"
+                            " {validate_canary_cmd}"
+                            " {seed_docs}").format(
+                                rsync_opt=rsync_opt,
+                                canary_opt=canary_opt,
+                                port=secret_port,
+                                rsync_cmd=rsync_cmd,
+                                validate_collections_cmd=validate_collections_cmd,
+                                validate_canary_cmd=validate_canary_cmd,
+                                seed_docs=seed_docs if loop_num == 1 else "")
+        ret, output = call_remote_operation(
+            local_ops,
+            options.remote_python,
+            script_name,
+            client_args,
+            remote_operation)
+        rsync_text = "rsync_data beforerecovery & " if options.rsync_data else ""
+        LOGGER.info("****%sstart mongod: %d %s****", rsync_text, ret, output)
+        if ret:
+            sys.exit(ret)
+
+        # Optionally, run local validation of collections.
+        if options.validate_collections == "local":
+            cmds = """
+                TestData = {};
+                TestData.skipValidationOnNamespaceNotFound = true;
+                load("jstests/hooks/run_validate_collections.js");"""
+            host_port = "{}:{}".format(mongod_host, secret_port)
+            ret, output = mongo_shell(options.mongo_path, mongo_repo_root_dir, host_port, cmds)
+            LOGGER.info("Collection validation: %d %s", ret, output)
+            if ret:
+                sys.exit(ret)
+
+        # Shutdown mongod on secret port.
+        remote_op = ("--remoteOperation"
+                     " --mongodPort {}"
+                     " shutdown_mongod").format(secret_port)
+        ret, output = call_remote_operation(
+            local_ops,
+            options.remote_python,
+            script_name,
+            client_args,
+            remote_op)
+        LOGGER.info("****shutdown_mongod: %d %s****", ret, output)
+        if ret:
+            sys.exit(ret)
+
+        # Since rsync requires Posix style paths, we do not use os.path.join to
+        # construct the rsync destination directory.
+        if rsync_cmd:
+            if options.backup_path_after:
+                rsync_dest = options.backup_path_after
+            else:
+                rsync_dest = "{}/data-afterrecovery".format(options.root_dir)
+            rsync_opt = " --rsyncDest {}".format(rsync_dest)
+
+        # Optionally, rsync the post-recovery database.
+        # Start monogd on the standard port.
+        # Replica sets are optionally used in this mode.
+        use_replica_set = "--useReplicaSet" if options.repl_set else ""
+        remote_op = ("--remoteOperation"
+                     " {}"
+                     " --mongodPort {}"
+                     " {}"
+                     " {}"
+                     " start_mongod").format(
+                         rsync_opt, standard_port, use_replica_set, rsync_cmd)
+        ret, output = call_remote_operation(
+            local_ops,
+            options.remote_python,
+            script_name,
+            client_args,
+            remote_op)
+        rsync_text = "rsync_data afterrecovery & " if options.rsync_data else ""
+        LOGGER.info("****%s start mongod: %d %s****", rsync_text, ret, output)
+        if ret:
+            sys.exit(ret)
+
+        # TODO SERVER-30802: Add CRUD & FSM clients
+
+        # Crash the server. A pre-crash canary document is optionally written to the DB.
+        crash_canary = {}
+        if options.canary:
+            canary_doc = {"x": time.time()}
+            orig_canary_doc = copy.deepcopy(canary_doc)
+            mongo_opts = get_mongo_client_args(options)
+            mongo = pymongo.MongoClient(
+                host=mongod_host, port=standard_port, **mongo_opts)
+            crash_canary["function"] = mongo_insert_canary
+            crash_canary["args"] = [
+                mongo,
+                options.db_name,
+                options.collection_name,
+                canary_doc]
+        crash_server(options, crash_canary, local_ops, script_name, client_args)
+        canary_doc = copy.deepcopy(orig_canary_doc)
+
+        test_time = int(time.time()) - start_time
+        LOGGER.info("****Completed test loop %d test time %d seconds****", loop_num, test_time)
+        if loop_num == options.num_loops or test_time >= options.test_time:
+            break
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()