summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJonathan Abrahams <jonathan@mongodb.com>2017-10-25 16:18:46 -0400
committerJonathan Abrahams <jonathan@mongodb.com>2017-10-25 16:18:46 -0400
commit3b5c492f3067ef2fbd33a631feded7b1e5224cbd (patch)
treeeffe7205b50c89f68141f9523ae346c1105cd9fd
parent4a03a53a6a420c0baf9fd687a447fd6d942a1c84 (diff)
downloadmongo-3b5c492f3067ef2fbd33a631feded7b1e5224cbd.tar.gz
SERVER-30802 Add CRUD & FSM clients to powertest.py
-rwxr-xr-xbuildscripts/launch_evergreen_ec2_instance.sh17
-rw-r--r--buildscripts/resmokeconfig/suites/with_external_server.yml8
-rw-r--r--etc/evergreen.yml151
-rw-r--r--jstests/concurrency/fsm_libs/cluster.js5
-rw-r--r--jstests/concurrency/fsm_libs/runner.js43
-rw-r--r--jstests/hooks/crud_client.js145
-rw-r--r--jstests/libs/fsm_serial_client.js17
-rwxr-xr-xpytests/powertest.py419
8 files changed, 672 insertions, 133 deletions
diff --git a/buildscripts/launch_evergreen_ec2_instance.sh b/buildscripts/launch_evergreen_ec2_instance.sh
index 871130191ae..7562e5ab79a 100755
--- a/buildscripts/launch_evergreen_ec2_instance.sh
+++ b/buildscripts/launch_evergreen_ec2_instance.sh
@@ -14,20 +14,25 @@ function _usage_ {
usage: $0 options
This script supports the following parameters for Windows & Linux platforms:
-k <ssh_key_id>, [REQUIRED] The ssh key id used to access the new AWS EC2 instance.
- -t <tag_name>, [OPTIONAL] The tag name of the new AWS EC2 instance.
-y <aws_ec2_yml>, [REQUIRED] YAML file name where to store the new AWS EC2 instance
information. This file will be used in etc/evergreen.yml for
macro expansion of variables used in other functions.
+ -s <security_group>, [OPTIONAL] The security group to be used for the new AWS EC2 instance.
+ To specify more than one group, invoke this option each time.
+ -t <tag_name>, [OPTIONAL] The tag name of the new AWS EC2 instance.
EOF
}
# Parse command line options
-while getopts "y:k:t:?" option
+while getopts "k:s:t:y:?" option
do
case $option in
k)
ssh_key_id=$OPTARG
;;
+ s)
+ sec_groups="$sec_groups $OPTARG"
+ ;;
t)
tag_name=$OPTARG
;;
@@ -51,6 +56,11 @@ if [ -z $ssh_key_id ]; then
exit 1
fi
+for sec_group in $sec_groups
+do
+ security_groups="$security_groups --securityGroup $sec_group"
+done
+
# Get the AMI information on the current host so we can launch a similar EC2 instance.
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html#instancedata-data-retrieval
aws_metadata_url="http://169.254.169.254/latest/meta-data"
@@ -133,7 +143,8 @@ aws_ec2=$(python buildscripts/aws_ec2.py \
--instanceType $instance_type \
--keyName $ssh_key_id \
--mode create \
- --tagName "$tag_name" \
+ $security_groups \
+ --tagName "$tag_name" \
--tagOwner "$USER" \
$block_device_option | tr -cd "[:print:]\n")
echo "Spawned new AMI EC2 instance: $aws_ec2"
diff --git a/buildscripts/resmokeconfig/suites/with_external_server.yml b/buildscripts/resmokeconfig/suites/with_external_server.yml
new file mode 100644
index 00000000000..a7ad1316cf7
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/with_external_server.yml
@@ -0,0 +1,8 @@
+test_kind: js_test
+
+# Use this suite to connect to a running mongod and specify resmoke.py options
+# '--shellConnPort' or '--shellConnString'.
+executor:
+ config:
+ shell_options:
+ readMode: commands \ No newline at end of file
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index ee2f832059a..0bd6879d3b6 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -146,16 +146,19 @@ variables:
# Templates used by powercycle
- &powercycle_remote_credentials
private_key_file: $(${posix_workdir})/powercycle.pem
- private_key_remote: ${kitchen_private_key}
- aws_key_remote: ${kitchen_aws_key}
- aws_secret_remote: ${kitchen_aws_secret}
+ private_key_remote: ${powercycle_private_key}
+ aws_key_remote: ${powercycle_aws_key}
+ aws_secret_remote: ${powercycle_aws_secret}
# TODO: Remove psutil from virtualenv_requirements when BUILD-3083 is done
- &powercycle_ec2_instance
aws_ec2_yml: aws_ec2.yml
remote_dir: powercycle
+ secret_port: "20001"
+ security_groups: mci powercycle_testing
ssh_identity: -i ${private_key_file}
- ssh_key_id: ${kitchen_ssh_key_id}
+ ssh_key_id: ${powercycle_ssh_key_id}
+ standard_port: "20000"
virtualenv_dir: venv_powercycle
virtualenv_requirements: psutil
@@ -175,10 +178,11 @@ variables:
ec2_artifacts: ${log_path} ${backup_path_after} ${backup_path_before}
program_options: --logLevel debug --backupPathBefore ${backup_path_before} --backupPathAfter ${backup_path_after}
connection_options: --sshUserHost ${ip_address} --sshConnection \"${ssh_identity} ${ssh_connection_options}\"
- test_options: --testLoops 25 --seedDocNum 10000 --rsync --validate remote --canary remote
+ test_options: --testLoops 25 --seedDocNum 10000 --rsync --validate local --canary remote
crash_options: --crashMethod internal --crashWaitTime 30 --jitterForCrashWaitTime 5
+ client_options: --numCrudClients 5 --numFsmClients 5
mongodb_options: --rootDir ${remote_dir}-${task_id} --mongodbBinDir ${remote_dir}
- mongod_options: --dbPath ${db_path} --logPath ${log_path}
+ mongod_options: --mongodUsablePorts ${standard_port} ${secret_port} --dbPath ${db_path} --logPath ${log_path}
mongod_extra_options: --mongodOptions \"--setParameter enableTestCommands=1\"
#######################################
@@ -830,56 +834,67 @@ functions:
fi
"run powercycle test" :
- command: shell.exec
- params:
- working_dir: src
- script: |
- set -o verbose
- set -o errexit
+ - command: shell.exec
+ params:
+ working_dir: src
+ script: |
+ set -o verbose
+ set -o errexit
- ${activate_virtualenv}
- if [ ! -z "${virtualenv_requirements}" ]; then
- easy_install ${virtualenv_requirements}
- fi
+ ${activate_virtualenv}
+ if [ ! -z "${virtualenv_requirements}" ]; then
+ pip install ${virtualenv_requirements}
+ fi
- ${set_sudo}
- if [ ! -z $sudo ]; then
- remote_sudo="--remoteSudo"
- fi
+ - command: shell.exec
+ type: test
+ params:
+ working_dir: src
+ script: |
+ set -o verbose
+ set -o errexit
+
+ ${activate_virtualenv}
+
+ ${set_sudo}
+ if [ ! -z $sudo ]; then
+ remote_sudo="--remoteSudo"
+ fi
- # The virtaulenv bin_dir is different for Linux and Windows
- bin_dir=$(find $VIRTUAL_ENV -name activate | sed -e "s,$VIRTUAL_ENV,,;s,activate,,;s,/,,g")
- cmds="source ${virtualenv_dir|venv}/$bin_dir/activate"
- cmds="$cmds; python -u"
- # The remote python operates in a virtualenv
- remote_python="--remotePython \"$cmds\""
+ # The virtaulenv bin_dir is different for Linux and Windows
+ bin_dir=$(find $VIRTUAL_ENV -name activate | sed -e "s,$VIRTUAL_ENV,,;s,activate,,;s,/,,g")
+ cmds="source ${virtualenv_dir|venv}/$bin_dir/activate"
+ cmds="$cmds; python -u"
+ # The remote python operates in a virtualenv
+ remote_python="--remotePython \"$cmds\""
- start_time=$(date +%s)
- status="\"pass\""
- failures=0
+ start_time=$(date +%s)
+ status="\"pass\""
+ failures=0
- set +o errexit
- eval python -u pytests/powertest.py \
- "${connection_options} \
- ${program_options} \
- $remote_sudo \
- $remote_python \
- ${test_options} \
- ${crash_options} \
- ${mongodb_options} \
- ${mongod_options} \
- ${mongod_extra_options}"
- exit_code=$?
+ set +o errexit
+ eval python -u pytests/powertest.py \
+ "${connection_options} \
+ ${program_options} \
+ $remote_sudo \
+ $remote_python \
+ ${test_options} \
+ ${crash_options} \
+ ${client_options} \
+ ${mongodb_options} \
+ ${mongod_options} \
+ ${mongod_extra_options}"
+ exit_code=$?
- # Create report.json
- end_time=$(date +%s)
- elapsed_secs=$((end_time-start_time))
- if [ $exit_code -ne 0 ]; then
- status="\"fail\""
- failures=1
- fi
- echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > report.json
- exit $exit_code
+ # Create report.json
+ end_time=$(date +%s)
+ elapsed_secs=$((end_time-start_time))
+ if [ $exit_code -ne 0 ]; then
+ status="\"fail\""
+ failures=1
+ fi
+ echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > report.json
+ exit $exit_code
"do multiversion setup" :
command: shell.exec
@@ -1087,11 +1102,17 @@ functions:
set -o errexit
${activate_virtualenv}
- easy_install --upgrade boto3
+ pip install boto3
+
+ for security_group in ${security_groups}
+ do
+ security_groups="$security_groups -s $security_group"
+ done
# Clone another instance of this host in EC2.
buildscripts/launch_evergreen_ec2_instance.sh \
-k ${ssh_key_id} \
+ $security_groups \
-t "AMI Evergreen ${task_id}" \
-y ${aws_ec2_yml}
@@ -1220,7 +1241,7 @@ functions:
cmds="$cmds; virtualenv --python \$python_loc --system-site-packages ${virtualenv_dir|venv}"
cmds="$cmds; activate=\$(find ${virtualenv_dir|venv} -name 'activate')"
cmds="$cmds; source \$activate"
- cmds="$cmds; easy_install ${virtualenv_requirements}"
+ cmds="$cmds; pip install ${virtualenv_requirements}"
ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \
--verbose \
@@ -1240,7 +1261,7 @@ functions:
if [ -z "${ec2_artifacts}" ]; then
exit 0
fi
- cmd="${tar|tar} czvf ec2_artifacts.tgz ${ec2_artifacts}"
+ cmd="${tar|tar} czf ec2_artifacts.tgz ${ec2_artifacts}"
ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \
--verbose \
@@ -1429,6 +1450,27 @@ post:
display_name: Remote EC2 Artifacts - Execution ${execution}
optional: true
+ # Gather and archive the local client logs.
+ - command: shell.exec
+ params:
+ working_dir: src
+ script: |
+ client_logs=$(ls crud*.log fsm*.log 2> /dev/null)
+ if [ ! -z "$client_logs" ]; then
+ ${tar|tar} czf client-logs.tgz $client_logs
+ fi
+ - command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/client-logs.tgz
+ remote_file: mongodb-mongo-master/${build_variant}/${revision}/client_logs/mongo-client-logs-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: Client logs - Execution ${execution}
+ optional: true
+
# Process and save coverage data.
- command: shell.exec
params:
@@ -4026,8 +4068,11 @@ tasks:
- command: expansions.update
<<: *powercycle_expansions
- func: "run powercycle test"
+ # Disable the CRUD & FSM clients for mmapv1.
+ # mongod will not start if it crashed mongod while creating a namespace (SERVER-26499).
vars:
<<: *powercycle_test
+ client_options: --numCrudClients 0 --numFsmClients 0
mongod_extra_options: --mongodOptions \"--setParameter enableTestCommands=1 --storageEngine mmapv1\"
- name: powercycle_WT
diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js
index 931e75a0af1..451de070f9d 100644
--- a/jstests/concurrency/fsm_libs/cluster.js
+++ b/jstests/concurrency/fsm_libs/cluster.js
@@ -617,7 +617,10 @@ var Cluster = function(options) {
var res = adminDB.runCommand({getCmdLineOpts: 1});
assert.commandWorked(res, 'failed to get command line options');
- var wiredTigerOptions = res.parsed.storage.wiredTiger || {};
+ var wiredTigerOptions = {};
+ if (res.parsed && res.parsed.storage) {
+ wiredTigerOptions = res.parsed.storage.wiredTiger || {};
+ }
var wiredTigerCollectionConfig = wiredTigerOptions.collectionConfig || {};
var wiredTigerConfigString = wiredTigerCollectionConfig.configString || '';
diff --git a/jstests/concurrency/fsm_libs/runner.js b/jstests/concurrency/fsm_libs/runner.js
index 4997a7bf775..cb5cb79f34e 100644
--- a/jstests/concurrency/fsm_libs/runner.js
+++ b/jstests/concurrency/fsm_libs/runner.js
@@ -116,7 +116,7 @@ var runner = (function() {
}
function validateCleanupOptions(options) {
- var allowedKeys = ['dropDatabaseBlacklist', 'keepExistingDatabases'];
+ var allowedKeys = ['dropDatabaseBlacklist', 'keepExistingDatabases', 'validateCollections'];
Object.keys(options).forEach(function(option) {
assert.contains(option,
@@ -136,6 +136,12 @@ var runner = (function() {
'expected keepExistingDatabases to be a boolean');
}
+ options.validateCollections =
+ options.hasOwnProperty('validateCollections') ? options.validateCollections : true;
+ assert.eq('boolean',
+ typeof options.validateCollections,
+ 'expected validateCollections to be a boolean');
+
return options;
}
@@ -441,7 +447,8 @@ var runner = (function() {
jsTest.log('End of schedule');
}
- function cleanupWorkload(workload, context, cluster, errors, header, dbHashBlacklist) {
+ function cleanupWorkload(
+ workload, context, cluster, errors, header, dbHashBlacklist, cleanupOptions) {
// Returns true if the workload's teardown succeeds and false if the workload's
// teardown fails.
@@ -458,7 +465,9 @@ var runner = (function() {
}
try {
- cluster.validateAllCollections(phase);
+ if (cleanupOptions.validateCollections) {
+ cluster.validateAllCollections(phase);
+ }
} catch (e) {
errors.push(new WorkloadFailure(
e.toString(), e.stack, 'main', header + ' validating collections'));
@@ -509,7 +518,8 @@ var runner = (function() {
errors,
maxAllowedThreads,
dbHashBlacklist,
- configServerData) {
+ configServerData,
+ cleanupOptions) {
var cleanup = [];
var teardownFailed = false;
var startTime = Date.now(); // Initialize in case setupWorkload fails below.
@@ -588,9 +598,13 @@ var runner = (function() {
} finally {
// Call each foreground workload's teardown function. After all teardowns have completed
// check if any of them failed.
- var cleanupResults =
- cleanup.map(workload => cleanupWorkload(
- workload, context, cluster, errors, 'Foreground', dbHashBlacklist));
+ var cleanupResults = cleanup.map(workload => cleanupWorkload(workload,
+ context,
+ cluster,
+ errors,
+ 'Foreground',
+ dbHashBlacklist,
+ cleanupOptions));
teardownFailed = cleanupResults.some(success => (success === false));
totalTime = Date.now() - startTime;
@@ -622,8 +636,8 @@ var runner = (function() {
validateExecutionOptions(executionMode, executionOptions);
Object.freeze(executionOptions); // immutable after validation (and normalization)
- Object.freeze(cleanupOptions); // immutable prior to validation
validateCleanupOptions(cleanupOptions);
+ Object.freeze(cleanupOptions); // immutable after validation (and normalization)
if (executionMode.composed) {
clusterOptions.sameDB = true;
@@ -743,7 +757,8 @@ var runner = (function() {
errors,
maxAllowedThreads,
dbHashBlacklist,
- configServerData);
+ configServerData,
+ cleanupOptions);
});
} finally {
// Set a flag so background threads know to terminate.
@@ -755,9 +770,13 @@ var runner = (function() {
} finally {
try {
// Call each background workload's teardown function.
- bgCleanup.forEach(
- bgWorkload => cleanupWorkload(
- bgWorkload, bgContext, cluster, errors, 'Background', dbHashBlacklist));
+ bgCleanup.forEach(bgWorkload => cleanupWorkload(bgWorkload,
+ bgContext,
+ cluster,
+ errors,
+ 'Background',
+ dbHashBlacklist,
+ cleanupOptions));
// TODO: Call cleanupWorkloadData() on background workloads here if no background
// workload teardown functions fail.
diff --git a/jstests/hooks/crud_client.js b/jstests/hooks/crud_client.js
new file mode 100644
index 00000000000..a20cce5da69
--- /dev/null
+++ b/jstests/hooks/crud_client.js
@@ -0,0 +1,145 @@
+// Basic CRUD client to provide load for Powercycle testing.
+
+'use strict';
+
+function randString(maxLength) {
+ maxLength = maxLength || 1024;
+ const randChars = "ABCDEFGHIKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ var rString = "";
+ for (var i = 0; i < Random.randInt(maxLength); i++) {
+ rString += randChars.charAt(Random.randInt(randChars.length));
+ }
+ return rString;
+}
+
+function weightedChoice(choices) {
+ var total = 0;
+ for (var choice in choices) {
+ total += choices[choice];
+ }
+ var ran = Random.randInt(total);
+ var upto = 0;
+ for (choice in choices) {
+ var weight = choices[choice];
+ if (upto + weight >= ran) {
+ return choice;
+ }
+ upto += weight;
+ }
+}
+
+var operations = {
+ "bulk insert": 15,
+ "count": 20,
+ "find": 15,
+ "remove multi": 15,
+ "remove one": 15,
+ "upsert one": 15,
+ "upsert multi": 5
+};
+
+Random.setRandomSeed();
+
+if (typeof TestData === "undefined") {
+ TestData = {};
+}
+var dbName = TestData.dbName || "test";
+var collectionName = TestData.collectionName || "crud";
+var bulkNum = TestData.bulkNum || 1000;
+var baseNum = TestData.baseNum || 100000;
+// Set numLoops <= 0 to have an infinite loop.
+var numLoops = TestData.numLoops || 0;
+
+print("****Starting CRUD client, namespace", dbName, collectionName, "numLoops", numLoops, "****");
+
+var coll = db.getSiblingDB(dbName)[collectionName];
+var writeConcern = TestData.writeConcern || {};
+if (Object.keys(writeConcern).length) {
+ coll.setWriteConcern = writeConcern;
+}
+
+var readConcern = TestData.readConcern || {};
+var noReadConcern = true;
+if (Object.keys(readConcern).length) {
+ noReadConcern = false;
+}
+
+coll.createIndex({x: 1});
+
+var shouldLoopForever = numLoops <= 0;
+while (shouldLoopForever || numLoops > 0) {
+ if (!shouldLoopForever) {
+ numLoops -= 1;
+ }
+
+ var info = db.hostInfo();
+ var serverStatus = db.serverStatus();
+ print("(" + collectionName + ") dbHostInfo status:",
+ info.ok,
+ serverStatus.version,
+ "uptime:",
+ serverStatus.uptime);
+ var match = Random.randInt(baseNum);
+ var matchQuery = {$gte: match, $lt: match + (baseNum * 0.01)};
+
+ var operation = weightedChoice(operations);
+
+ if (operation == "upsert multi") {
+ var updateOpts = {upsert: true, multi: true};
+ print("(" + collectionName + ") Upsert multi docs",
+ tojsononeline(matchQuery),
+ tojsononeline(updateOpts),
+ tojsononeline(coll.update(
+ {x: matchQuery}, {$inc: {x: baseNum}, $set: {n: "hello"}}, updateOpts)));
+ } else if (operation == "upsert one") {
+ var updateOpts = {upsert: true, multi: false};
+ print("(" + collectionName + ") Upsert single doc",
+ match,
+ tojsononeline(updateOpts),
+ tojsononeline(
+ coll.update({x: match}, {$inc: {x: baseNum}, $set: {n: "hello"}}, updateOpts)));
+ } else if (operation == "bulk insert") {
+ var bulk = coll.initializeUnorderedBulkOp();
+ for (var i = 0; i < bulkNum; i++) {
+ bulk.insert({x: (match + i) % baseNum, doc: randString()});
+ }
+ print("(" + collectionName + ") Bulk insert",
+ bulkNum,
+ "docs",
+ tojsononeline(writeConcern),
+ tojsononeline(bulk.execute(writeConcern)));
+ } else if (operation == "count") {
+ var countOpts = {count: collectionName, query: {x: matchQuery}};
+ if (!noReadConcern) {
+ countOpts.readConcern = readConcern;
+ }
+ print("(" + collectionName + ") Count docs",
+ tojsononeline(matchQuery),
+ tojsononeline(readConcern),
+ tojsononeline(db.runCommand(countOpts)));
+ } else if (operation == "find") {
+ var findOpts = {find: collectionName, singleBatch: true, filter: {x: matchQuery}};
+ if (!noReadConcern) {
+ findOpts.readConcern = readConcern;
+ }
+ print("(" + collectionName + ") Find docs",
+ tojsononeline(matchQuery),
+ tojsononeline(readConcern),
+ "find status",
+ db.runCommand(findOpts).ok);
+ } else if (operation == "remove multi") {
+ var removeOpts = {};
+ var removeQuery = {x: matchQuery, justOne: false};
+ print("(" + collectionName + ") Remove docs",
+ tojsononeline(removeQuery),
+ tojsononeline(removeOpts),
+ tojsononeline(coll.remove(removeQuery, removeOpts)));
+ } else if (operation == "remove one") {
+ var removeOpts = {};
+ var removeQuery = {x: match, justOne: true};
+ print("(" + collectionName + ") Remove docs",
+ tojsononeline(removeQuery),
+ tojsononeline(removeOpts),
+ tojsononeline(coll.remove(removeQuery, removeOpts)));
+ }
+}
diff --git a/jstests/libs/fsm_serial_client.js b/jstests/libs/fsm_serial_client.js
new file mode 100644
index 00000000000..6aa246c0abc
--- /dev/null
+++ b/jstests/libs/fsm_serial_client.js
@@ -0,0 +1,17 @@
+// This is the template file used in Powercycle testing for launching FSM Serial clients.
+'use strict';
+
+load('jstests/concurrency/fsm_libs/runner.js');
+
+var workloadDir = 'jstests/concurrency/fsm_workloads';
+
+var workloadList = TestData.workloadFiles || ls(workloadDir);
+var dbNamePrefix = TestData.dbNamePrefix || '';
+var fsmDbBlacklist = TestData.fsmDbBlacklist || [];
+var validateCollectionsOnCleanup = TestData.validateCollections;
+
+runWorkloadsSerially(workloadList, {}, {dbNamePrefix: dbNamePrefix}, {
+ keepExistingDatabases: true,
+ dropDatabaseBlacklist: fsmDbBlacklist,
+ validateCollections: validateCollectionsOnCleanup
+});
diff --git a/pytests/powertest.py b/pytests/powertest.py
index 49ce400d589..be18ebeabe2 100755
--- a/pytests/powertest.py
+++ b/pytests/powertest.py
@@ -2,11 +2,12 @@
"""Powercycle test
-Tests robustness of mongod to survice multiple powercycle events.
+Tests robustness of mongod to survive multiple powercycle events.
"""
from __future__ import print_function
+import atexit
import collections
import copy
import datetime
@@ -101,6 +102,36 @@ LOGGER = logging.getLogger(__name__)
This script will either download a MongoDB tarball or use an existing setup. """
+def exit_handler():
+ """Exit handler, deletes all named temporary files."""
+ LOGGER.debug("Exit handler invoked, cleaning up temporary files")
+ try:
+ NamedTempFile.delete_all()
+ except:
+ pass
+
+
+def kill_processes(pids, kill_children=True):
+ """Kill a list of processes and optionally it's children."""
+ for pid in pids:
+ LOGGER.debug("Killing process with pid %d", pid)
+ try:
+ proc = psutil.Process(pid)
+ except psutil.NoSuchProcess:
+ LOGGER.error("Could not kill process with pid %d, as it no longer exists", pid)
+ continue
+ if kill_children:
+ child_procs = proc.children(recursive=True)
+ child_pids = []
+ for child in child_procs:
+ child_pids.append(child.pid)
+ kill_processes(child_pids, kill_children=False)
+ try:
+ proc.kill()
+ except psutil.NoSuchProcess:
+ LOGGER.error("Could not kill process with pid %d, as it no longer exists", pid)
+
+
def get_extension(filename):
"""Returns the extension of a file."""
return os.path.splitext(filename)[-1]
@@ -140,40 +171,73 @@ def executable_exists_in_path(executable):
return distutils.spawn.find_executable(executable) is not None
+def create_temp_executable_file(cmds):
+ """Creates an executable temporary file containing 'cmds'. Returns file name."""
+ temp_file_name = NamedTempFile.create(suffix=".sh")
+ with NamedTempFile.get(temp_file_name) as temp_file:
+ temp_file.write(cmds)
+ os_st = os.stat(temp_file_name)
+ os.chmod(temp_file_name, os_st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+ return temp_file_name
+
+
+def start_cmd(cmd, use_file=False):
+ """Starts command and returns pid from Popen"""
+
+ orig_cmd = ""
+ # Multi-commands need to be written to a temporary file to execute on Windows.
+ # This is due to complications with invoking Bash in Windows.
+ if use_file:
+ orig_cmd = cmd
+ temp_file = create_temp_executable_file(cmd)
+ # The temporary file name will have '\' on Windows and needs to be converted to '/'.
+ cmd = "bash -c {}".format(temp_file.replace("\\", "/"))
+
+ # If 'cmd' is specified as a string, convert it to a list of strings.
+ if isinstance(cmd, str):
+ cmd = shlex.split(cmd)
+
+ if use_file:
+ LOGGER.debug("Executing '%s', tempfile contains: %s", cmd, orig_cmd)
+ else:
+ LOGGER.debug("Executing '%s'", cmd)
+
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+ return proc.pid
+
+
def execute_cmd(cmd, use_file=False):
- """Executes command and returns return_code and output from command"""
+ """Executes command and returns return_code, output from command"""
orig_cmd = ""
# Multi-commands need to be written to a temporary file to execute on Windows.
# This is due to complications with invoking Bash in Windows.
if use_file:
orig_cmd = cmd
- with tempfile.NamedTemporaryFile(suffix=".sh", delete=False) as temp_file:
- temp_file.write(cmd)
- os_st = os.stat(temp_file.name)
- os.chmod(temp_file.name, os_st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+ temp_file = create_temp_executable_file(cmd)
# The temporary file name will have '\' on Windows and needs to be converted to '/'.
- cmd = "bash -c {}".format(temp_file.name.replace("\\", "/"))
+ cmd = "bash -c {}".format(temp_file.replace("\\", "/"))
# If 'cmd' is specified as a string, convert it to a list of strings.
if isinstance(cmd, str):
cmd = shlex.split(cmd)
if use_file:
- LOGGER.info("Executing '%s', tempfile contains: %s", cmd, orig_cmd)
+ LOGGER.debug("Executing '%s', tempfile contains: %s", cmd, orig_cmd)
else:
- LOGGER.info("Executing '%s'", cmd)
+ LOGGER.debug("Executing '%s'", cmd)
try:
- proc = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output, _ = proc.communicate()
error_code = proc.returncode
if error_code:
output = "Error executing cmd {}: {}".format(cmd, output)
finally:
if use_file:
- os.remove(temp_file.name)
+ os.remove(temp_file)
+
return error_code, output
@@ -202,23 +266,44 @@ def parse_options(options):
return options_map
-def download_file(url, file_name):
+def download_file(url, file_name, download_retries=5):
"""Returns True if download was successful. Raises error if download fails."""
LOGGER.info("Downloading %s to %s", url, file_name)
- with requests.Session() as session:
- adapter = requests.adapters.HTTPAdapter(max_retries=5)
- session.mount(url, adapter)
- response = session.get(url, stream=True)
- response.raise_for_status()
-
- with open(file_name, "wb") as file_handle:
- for block in response.iter_content(1024):
- file_handle.write(block)
+ while download_retries > 0:
- adapter.close()
+ with requests.Session() as session:
+ adapter = requests.adapters.HTTPAdapter(max_retries=download_retries)
+ session.mount(url, adapter)
+ response = session.get(url, stream=True)
+ response.raise_for_status()
- return True
+ with open(file_name, "wb") as file_handle:
+ try:
+ for block in response.iter_content(1024 * 1000):
+ file_handle.write(block)
+ except requests.exceptions.ChunkedEncodingError as err:
+ download_retries -= 1
+ if download_retries == 0:
+ raise Exception("Incomplete download for URL {}: {}".format(url, err))
+ continue
+
+ # Check if file download was completed.
+ if "Content-length" in response.headers:
+ url_content_length = int(response.headers["Content-length"])
+ file_size = os.path.getsize(file_name)
+ # Retry download if file_size has an unexpected size.
+ if url_content_length != file_size:
+ download_retries -= 1
+ if download_retries == 0:
+ raise Exception("Downloaded file size ({} bytes) doesn't match content length"
+ "({} bytes) for URL {}".format(
+ file_size, url_content_length, url))
+ continue
+
+ return True
+
+ raise Exception("Unknown download problem for {} to file {}".format(url, file_name))
def install_tarball(tarball, root_dir):
@@ -361,7 +446,7 @@ def install_mongod(bin_dir=None, tarball_url="latest", root_dir=None):
# Symlink the bin dir from the tarball to 'root_bin_dir'.
# Since get_bin_dir returns an abolute path, we need to remove 'root_dir'
tarball_bin_dir = get_bin_dir(root_dir).replace("{}/".format(root_dir), "")
- LOGGER.info("Symlink %s to %s", tarball_bin_dir, root_bin_dir)
+ LOGGER.debug("Symlink %s to %s", tarball_bin_dir, root_bin_dir)
symlink_dir(tarball_bin_dir, root_bin_dir)
@@ -380,6 +465,40 @@ def call_remote_operation(local_ops, remote_python, script_name, client_args, op
return ret, output
+class NamedTempFile(object):
+ """Class to control temporary files."""
+
+ _FILE_MAP = {}
+
+ @classmethod
+ def create(cls, suffix=""):
+ """Creates a temporary file and returns the file name."""
+ temp_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
+ cls._FILE_MAP[temp_file.name] = temp_file
+ return temp_file.name
+
+ @classmethod
+ def get(cls, name):
+ """Gets temporary file object. Raises an exception if the file is unknown."""
+ if name not in cls._FILE_MAP:
+ raise Exception("Unknown temporary file {}.".format(name))
+ return cls._FILE_MAP[name]
+
+ @classmethod
+ def delete(cls, name):
+ """Deletes temporary file. Raises an exception if the file is unknown."""
+ if name not in cls._FILE_MAP:
+ raise Exception("Unknown temporary file {}.".format(name))
+ os.remove(name)
+ del cls._FILE_MAP[name]
+
+ @classmethod
+ def delete_all(cls):
+ """Deletes all temporary files."""
+ for name in list(cls._FILE_MAP):
+ cls.delete(name)
+
+
class ProcessControl(object):
""" Process control class.
@@ -430,14 +549,15 @@ class ProcessControl(object):
return True
return False
- def terminate(self):
- """ Terminates all running processes that match the list of pids. """
+ def kill(self):
+ """ Kills all running processes that match the list of pids. """
if self.is_running():
for proc in self.get_procs():
try:
- proc.terminate()
+ proc.kill()
except psutil.NoSuchProcess:
- LOGGER.info("Could not terminate pid %d, process no longer exists", proc.pid)
+ LOGGER.info("Could not kill process with pid %d, as it no longer exists",
+ proc.pid)
class WindowsService(object):
@@ -611,7 +731,7 @@ class PosixService(object):
def stop(self):
""" Stop process. Returns (code, output) tuple. """
proc = ProcessControl(name=self.bin_name)
- proc.terminate()
+ proc.kill()
self.pids = []
return 0, None
@@ -907,11 +1027,7 @@ def internal_crash(use_sudo=False):
# Windows does not have a way to immediately crash itself. It's
# better to use an external mechanism instead.
if _IS_WINDOWS:
- # Sleep after issuing shutdown, to prevent the 'client' side script
- # continuing, as shutdown is no immediate.
- cmds = """
- shutdown /r /f /t 0 ;
- sleep 10"""
+ cmds = "shutdown /r /f /t 0"
ret, output = execute_cmd(cmds, use_file=True)
return ret, output
else:
@@ -937,7 +1053,7 @@ def internal_crash(use_sudo=False):
return 1, "Crash did not occur"
-def crash_server(options, crash_canary, local_ops, script_name, client_args):
+def crash_server(options, crash_canary, canary_port, local_ops, script_name, client_args):
""" Crashes server and optionally writes canary doc before crash. """
crash_wait_time = options.crash_wait_time + random.randint(0, options.crash_wait_time_jitter)
@@ -962,7 +1078,8 @@ def crash_server(options, crash_canary, local_ops, script_name, client_args):
if options.canary == "remote":
# The crash canary function executes remotely, only if the
# crash_method is 'internal'.
- canary = "--docForCanary \"{}\"".format(crash_canary["args"][3])
+ canary = "--mongodPort {} --docForCanary \"{}\"".format(
+ canary_port, crash_canary["args"][3])
canary_cmd = "insert_canary"
else:
canary = ""
@@ -1011,7 +1128,7 @@ def get_mongo_client_args(options):
""" Returns keyword arg dict used in PyMongo client. """
mongo_args = {}
# Set the writeConcern
- mongo_args = options.write_concern
+ mongo_args = yaml.safe_load(options.write_concern)
# Set the readConcernLevel
if options.read_concern_level:
mongo_args["readConcernLevel"] = options.read_concern_level
@@ -1146,9 +1263,58 @@ def mongo_insert_canary(mongo, db_name, coll_name, doc):
return 0 if res.inserted_id else 1
+def new_resmoke_config(config_file, new_config_file, test_data):
+ """ Creates 'new_config_file', from 'config_file', with an update from 'test_data'. """
+ new_config = {
+ "executor": {
+ "config": {
+ "shell_options": {
+ "global_vars": {
+ "TestData": test_data
+ }
+ }
+ }
+ }
+ }
+ with open(config_file, "r") as yaml_stream:
+ config = yaml.load(yaml_stream)
+ config.update(new_config)
+ with open(new_config_file, "w") as yaml_stream:
+ yaml.safe_dump(config, yaml_stream)
+
+
+def resmoke_client(work_dir,
+ mongo_path,
+ host_port,
+ js_test,
+ resmoke_suite,
+ no_wait=False,
+ log_file=None):
+ """Starts resmoke client from work_dir, connecting to host_port and executes js_test."""
+ log_output = "2>& 1 | tee -a {}".format(log_file) if log_file else ""
+ cmds = ("cd {} ; "
+ "python buildscripts/resmoke.py "
+ "--mongo {} "
+ "--suites {} "
+ "--shellConnString mongodb://{} "
+ "--continueOnFailure "
+ "{} "
+ "{}".format(
+ work_dir, mongo_path, resmoke_suite, host_port, js_test, log_output))
+ ret, output, pid = None, None, None
+ if no_wait:
+ pid = start_cmd(cmds, use_file=True)
+ else:
+ ret, output = execute_cmd(cmds, use_file=True)
+ return ret, output, pid
+
+
def main():
""" Main program. """
+
+ atexit.register(exit_handler)
+
parser = optparse.OptionParser(usage="""
%prog [options]
@@ -1179,6 +1345,7 @@ Examples:
crash_options = optparse.OptionGroup(parser, "Crash Options")
mongodb_options = optparse.OptionGroup(parser, "MongoDB Options")
mongod_options = optparse.OptionGroup(parser, "mongod Options")
+ client_options = optparse.OptionGroup(parser, "Client Options")
program_options = optparse.OptionGroup(parser, "Program Options")
# Test options
@@ -1199,18 +1366,6 @@ Examples:
" which are added to '{}'".format(default_ssh_connection_options),
default=None)
- test_options.add_option("--mongoPath",
- dest="mongo_path",
- help="Path to mongo (shell) executable, if unspecifed, mongo client"
- " is launched from $PATH",
- default="mongo")
-
- test_options.add_option("--mongoRepoRootDir",
- dest="mongo_repo_root_dir",
- help="Root directory of mongoDB repository, defaults to current"
- " directory.",
- default=None)
-
test_options.add_option("--testLoops",
dest="num_loops",
help="Number of powercycle loops to run [default: %default]",
@@ -1400,6 +1555,60 @@ Examples:
" 'source venv/bin/activate; python'",
default="python")
+ # Client options
+ mongo_path = distutils.spawn.find_executable(
+ "mongo", os.getcwd() + os.pathsep + os.environ["PATH"])
+ if mongo_path:
+ mongo_path = os.path.abspath(mongo_path)
+ client_options.add_option("--mongoPath",
+ dest="mongo_path",
+ help="Path to mongo (shell) executable, if unspecifed, mongo client"
+ " is launched from the current directory.",
+ default=mongo_path)
+
+ client_options.add_option("--mongoRepoRootDir",
+ dest="mongo_repo_root_dir",
+ help="Root directory of mongoDB repository, defaults to current"
+ " directory.",
+ default=None)
+
+ client_options.add_option("--crudClient",
+ dest="crud_client",
+ help="The path to the CRUD client script on the local host"
+ " [default: '%default'].",
+ default="jstests/hooks/crud_client.js")
+
+ client_options.add_option("--configCrudClient",
+ dest="config_crud_client",
+ help="The path to the CRUD client configuration YML file on the"
+ " local host. This is the resmoke.py suite file. If"
+ " unspecified, a default configuration will be used that"
+ " provides a mongo (shell) DB connection to a running mongod.",
+ default=None)
+
+ client_options.add_option("--numCrudClients",
+ dest="num_crud_clients",
+ help="The number of concurrent CRUD clients to run"
+ " [default: '%default'].",
+ type="int",
+ default=1)
+
+ client_options.add_option("--numFsmClients",
+ dest="num_fsm_clients",
+ help="The number of concurrent FSM clients to run"
+ " [default: '%default'].",
+ type="int",
+ default=0)
+
+ client_options.add_option("--fsmWorkloadFiles",
+ dest="fsm_workload_files",
+ help="A list of the FSM workload files to execute. More than one"
+ " file can be specified either in a comma-delimited string,"
+ " or by specifying this option more than once. If unspecified,"
+ " then all FSM workload files are executed.",
+ action="append",
+ default=[])
+
# Program options
program_options.add_option("--remoteSudo",
dest="remote_sudo",
@@ -1458,6 +1667,7 @@ Examples:
parser.add_option_group(test_options)
parser.add_option_group(crash_options)
+ parser.add_option_group(client_options)
parser.add_option_group(mongodb_options)
parser.add_option_group(mongod_options)
parser.add_option_group(program_options)
@@ -1486,7 +1696,7 @@ Examples:
mongod_options_map = parse_options(options.mongod_options)
# Error out earlier if these options are not properly specified
- options.write_concern = yaml.safe_load(options.write_concern)
+ write_concern = yaml.safe_load(options.write_concern)
options.canary_doc = yaml.safe_load(options.canary_doc)
# Invoke remote_handler if remote_operation is specified.
@@ -1511,19 +1721,41 @@ Examples:
rsync_cmd = ""
rsync_opt = ""
- # Setup the mongo_repo_root
- mongo_repo_root_dir = "." if not options.mongo_repo_root_dir else options.mongo_repo_root_dir
+ # Setup the mongo client, mongo_path is required if there are local clients.
+ if (options.num_crud_clients > 0 or
+ options.num_fsm_clients > 0 or
+ options.validate_collections == "local"):
+ if options.mongo_path is None:
+ LOGGER.error("mongoPath must be specified")
+ sys.exit(1)
+ elif not os.path.isfile(options.mongo_path):
+ LOGGER.error("mongoPath %s does not exist", options.mongo_path)
+ sys.exit(1)
+
+ # Setup the CRUD & FSM clients.
+ with_external_server = "buildscripts/resmokeconfig/suites/with_external_server.yml"
+ config_crud_client = options.config_crud_client
+ fsm_client = "jstests/libs/fsm_serial_client.js"
+ fsm_workload_files = []
+ for fsm_workload_file in options.fsm_workload_files:
+ fsm_workload_files += fsm_workload_file.replace(" ", "").split(",")
+
+ # Setup the mongo_repo_root.
+ if options.mongo_repo_root_dir:
+ mongo_repo_root_dir = options.mongo_repo_root_dir
+ else:
+ mongo_repo_root_dir = os.getcwd()
if not os.path.isdir(mongo_repo_root_dir):
LOGGER.error("mongoRepoRoot %s does not exist", mongo_repo_root_dir)
sys.exit(1)
- # Setup the validate_collections option
+ # Setup the validate_collections option.
if options.validate_collections == "remote":
validate_collections_cmd = "validate_collections"
else:
validate_collections_cmd = ""
- # Setup the validate_canary option
+ # Setup the validate_canary option.
if options.canary and "nojournal" in mongod_options_map:
LOGGER.error("Cannot create and validate canary documents if the mongod option"
" '--nojournal' is used.")
@@ -1666,12 +1898,17 @@ Examples:
# Optionally, run local validation of collections.
if options.validate_collections == "local":
- cmds = """
- TestData = {};
- TestData.skipValidationOnNamespaceNotFound = true;
- load("jstests/hooks/run_validate_collections.js");"""
host_port = "{}:{}".format(mongod_host, secret_port)
- ret, output = mongo_shell(options.mongo_path, mongo_repo_root_dir, host_port, cmds)
+ new_config_file = NamedTempFile.create(suffix=".yml")
+ test_data = {"skipValidationOnNamespaceNotFound": True}
+ new_resmoke_config(with_external_server, new_config_file, test_data)
+ ret, output, _ = resmoke_client(
+ mongo_repo_root_dir,
+ options.mongo_path,
+ host_port,
+ "jstests/hooks/run_validate_collections.js",
+ new_config_file)
+ NamedTempFile.delete(new_config_file)
LOGGER.info("Collection validation: %d %s", ret, output)
if ret:
sys.exit(ret)
@@ -1721,7 +1958,57 @@ Examples:
if ret:
sys.exit(ret)
- # TODO SERVER-30802: Add CRUD & FSM clients
+ # Start CRUD clients
+ crud_pids = []
+ if options.num_crud_clients > 0:
+ host_port = "{}:{}".format(mongod_host, standard_port)
+ test_data = {"dbName": options.db_name}
+ if options.read_concern_level:
+ test_data["readConcern"] = {"level": options.read_concern_level}
+ if write_concern:
+ test_data["writeConcern"] = write_concern
+
+ for i in xrange(options.num_crud_clients):
+ crud_config_file = NamedTempFile.create(suffix=".yml")
+ test_data["collectionName"] = "{}-{}".format(options.collection_name, i)
+ new_resmoke_config(with_external_server, crud_config_file, test_data)
+ _, _, pid = resmoke_client(
+ work_dir=mongo_repo_root_dir,
+ mongo_path=options.mongo_path,
+ host_port=host_port,
+ js_test=options.crud_client,
+ resmoke_suite=crud_config_file,
+ no_wait=True,
+ log_file="crud_{}.log".format(i))
+ crud_pids.append(pid)
+
+ LOGGER.info(
+ "****Started %d CRUD client(s) %s****", options.num_crud_clients, crud_pids)
+
+ # Start FSM clients
+ fsm_pids = []
+ if options.num_fsm_clients > 0:
+ test_data = {"fsmDbBlacklist": [options.db_name]}
+ if fsm_workload_files:
+ test_data["workloadFiles"] = fsm_workload_files
+
+ for i in xrange(options.num_fsm_clients):
+ fsm_config_file = NamedTempFile.create(suffix=".yml")
+ test_data["dbNamePrefix"] = "fsm-{}".format(i)
+ # Do collection validation only for the first FSM client.
+ test_data["validateCollections"] = True if i == 0 else False
+ new_resmoke_config(with_external_server, fsm_config_file, test_data)
+ _, _, pid = resmoke_client(
+ work_dir=mongo_repo_root_dir,
+ mongo_path=options.mongo_path,
+ host_port=host_port,
+ js_test=fsm_client,
+ resmoke_suite=fsm_config_file,
+ no_wait=True,
+ log_file="fsm_{}.log".format(i))
+ fsm_pids.append(pid)
+
+ LOGGER.info("****Started %d FSM client(s) %s****", options.num_fsm_clients, fsm_pids)
# Crash the server. A pre-crash canary document is optionally written to the DB.
crash_canary = {}
@@ -1729,17 +2016,21 @@ Examples:
canary_doc = {"x": time.time()}
orig_canary_doc = copy.deepcopy(canary_doc)
mongo_opts = get_mongo_client_args(options)
- mongo = pymongo.MongoClient(
- host=mongod_host, port=standard_port, **mongo_opts)
+ mongo = pymongo.MongoClient(host=mongod_host, port=standard_port, **mongo_opts)
crash_canary["function"] = mongo_insert_canary
crash_canary["args"] = [
mongo,
options.db_name,
options.collection_name,
canary_doc]
- crash_server(options, crash_canary, local_ops, script_name, client_args)
+ crash_server(options, crash_canary, standard_port, local_ops, script_name, client_args)
+ # Wait a bit after sending command to crash the server to avoid connecting to the
+ # server before the actual crash occurs.
+ time.sleep(10)
canary_doc = copy.deepcopy(orig_canary_doc)
+ kill_processes(crud_pids + fsm_pids)
+
test_time = int(time.time()) - start_time
LOGGER.info("****Completed test loop %d test time %d seconds****", loop_num, test_time)
if loop_num == options.num_loops or test_time >= options.test_time: