From 368ad3fee1accc4b5a9c384036dd3ee41b272a3b Mon Sep 17 00:00:00 2001 From: Jonathan Abrahams Date: Mon, 23 Jul 2018 14:40:58 -0400 Subject: SERVER-33695 Include the loop name in the before and after recovery files in powertest.py (cherry picked from commit abc9ecda03f70ad38eef15b7c8dd394811ba7cff) --- etc/evergreen.yml | 6 ++++-- pytests/powertest.py | 41 +++++++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index a4f79e0a155..df19201fecf 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -183,6 +183,8 @@ variables: value: ${remote_dir}/afterrecovery - key: backup_path_before value: ${remote_dir}/beforerecovery + - key: backup_artifacts + value: ${remote_dir}/afterrecovery* ${remote_dir}/beforerecovery* - key: db_path value: /data/db - key: log_path @@ -191,8 +193,8 @@ variables: value: powercycle_exit.yml - &powercycle_test - ec2_artifacts: ${log_path} ${db_path} ${backup_path_after} ${backup_path_before} - program_options: --exitYamlFile=${exit_file} --logLevel=info --backupPathBefore=${backup_path_before} --backupPathAfter=${backup_path_after} + ec2_artifacts: ${log_path} ${db_path} ${backup_artifacts} + program_options: --exitYamlFile=${exit_file} --logLevel=info --backupPathBefore=${backup_path_before} --backupPathAfter=${backup_path_after} connection_options: --sshUserHost=${private_ip_address} --sshConnection=\"${ssh_identity} ${ssh_connection_options}\" test_options: --testLoops=15 --seedDocNum=10000 --rsync --rsyncExcludeFiles=diagnostic.data/metrics.interim* --validate=local --canary=local crash_options: --crashMethod=internal --crashOption=${windows_crash_cmd} --crashWaitTime=45 --jitterForCrashWaitTime=5 --instanceId=${instance_id} diff --git a/pytests/powertest.py b/pytests/powertest.py index b6ca3902f76..2bdf196d2d8 100755 --- a/pytests/powertest.py +++ b/pytests/powertest.py @@ -107,7 +107,7 @@ if _IS_WINDOWS: _try_import("win32serviceutil") -__version__ = "0.1" +__version__ = "1.0" LOGGER = logging.getLogger(__name__) @@ -1268,8 +1268,14 @@ def remote_handler(options, operations): ret = wait_for_mongod_shutdown(mongod) elif operation == "rsync_data": - ret, output = rsync(options.db_path, options.rsync_dest, options.rsync_exclude_files) - LOGGER.info(output) + rsync_dir, new_rsync_dir = options.rsync_dest + ret, output = rsync(options.db_path, rsync_dir, options.rsync_exclude_files) + if output: + LOGGER.info(output) + # Rename the rsync_dir only if it has a different name than new_rsync_dir. + if ret == 0 and rsync_dir != new_rsync_dir: + LOGGER.info("Renaming directory {} to {}".format(rsync_dir, new_rsync_dir)) + os.rename(rsync_dir, new_rsync_dir) elif operation == "seed_docs": mongo = pymongo.MongoClient(**mongo_client_opts) @@ -1321,6 +1327,11 @@ def remote_handler(options, operations): return 0 +def get_backup_path(path, loop_num): + """Return the backup path based on the loop_num.""" + return re.sub("-{}$".format(loop_num - 1), "-{}".format(loop_num), path) + + def rsync(src_dir, dest_dir, exclude_files=None): """ Rsync 'src_dir' to 'dest_dir'. """ # Note rsync on Windows requires a Unix-style directory. @@ -2114,16 +2125,11 @@ Examples: # Remote options, include commands and options sent from client to server under test. # These are 'internal' options, not meant to be directly specifed. # More than one remote operation can be provided and they are specified in the program args. - program_options.add_option("--remoteOperation", - dest="remote_operation", - help=optparse.SUPPRESS_HELP, - action="store_true", - default=False) + program_options.add_option("--remoteOperation", dest="remote_operation", + help=optparse.SUPPRESS_HELP, action="store_true", default=False) - program_options.add_option("--rsyncDest", - dest="rsync_dest", - help=optparse.SUPPRESS_HELP, - default=None) + program_options.add_option("--rsyncDest", dest="rsync_dest", nargs=2, + help=optparse.SUPPRESS_HELP, default=None) parser.add_option_group(test_options) parser.add_option_group(crash_options) @@ -2257,6 +2263,9 @@ Examples: backup_path_after = options.backup_path_after if not backup_path_after: backup_path_after = "{}/data-afterrecovery".format(options.root_dir) + # Set the first backup directory, for loop 1. + backup_path_before = "{}-1".format(backup_path_before) + backup_path_after = "{}-1".format(backup_path_after) else: rsync_cmd = "" rsync_opt = "" @@ -2451,7 +2460,9 @@ Examples: # Since rsync requires Posix style paths, we do not use os.path.join to # construct the rsync destination directory. if rsync_cmd: - rsync_opt = "--rsyncDest {}".format(backup_path_before) + new_path_dir = get_backup_path(backup_path_before, loop_num) + rsync_opt = "--rsyncDest {} {}".format(backup_path_before, new_path_dir) + backup_path_before = new_path_dir # Optionally, rsync the pre-recovery database. # Start monogd on the secret port. @@ -2530,7 +2541,9 @@ Examples: # Since rsync requires Posix style paths, we do not use os.path.join to # construct the rsync destination directory. if rsync_cmd: - rsync_opt = "--rsyncDest {}".format(backup_path_after) + new_path_dir = get_backup_path(backup_path_after, loop_num) + rsync_opt = "--rsyncDest {} {}".format(backup_path_after, new_path_dir) + backup_path_after = new_path_dir # Optionally, rsync the post-recovery database. # Start monogd on the standard port. -- cgit v1.2.1