QPID-5361: Added a bash script which allows for transactional soak testing.

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1564893 13f79535-47bb-0310-9956-ffa450edef68

1 files changed, 234 insertions, 0 deletions

diff --git a/qpid/cpp/src/tests/linearstore/tx-test-soak.sh b/qpid/cpp/src/tests/linearstore/tx-test-soak.sh
new file mode 100755
index 0000000000..072b606ecb
--- /dev/null
+++ b/qpid/cpp/src/tests/linearstore/tx-test-soak.sh
@@ -0,0 +1,234 @@
+#! /bin/bash
+
+# tx-test-soak
+#
+# Basic test methodology:
+# 1. Start broker
+# 2. Run qpid-txtest against broker using randomly generated parameters
+# 3. After some time, kill the broker using SIGKILL
+# 4. Restart broker, recover messages
+# 5. Run qpid-txtest against broker in check mode, which checks that all expected messages are present.
+# 6. Wash, rinse, repeat... The number of runs is determined by ${NUM_RUNS}
+
+NUM_RUNS=1000
+BASE_DIR=${HOME}/RedHat
+CMAKE_BUILD_DIR=${BASE_DIR}/q.cm
+
+# Infrequently adjusted
+RESULT_BASE_DIR_PREFIX=${BASE_DIR}/results.tx-test-soak
+RECOVER_TIME_PER_QUEUE=1
+STORE_MODULE="linearstore.so"
+BROKER_LOG_LEVEL="info+"
+BROKER_MANAGEMENT="no" # "no" or "yes"
+TRUNCATE_INTERVAL=10
+MAX_DISK_PERC_USED=90
+
+# Consts (don't adjust these...)
+export BASE_DIR
+RELATIVE_BASE_DIR=`python -c "import os,os.path; print os.path.relpath(os.environ['BASE_DIR'], os.environ['PWD'])"`
+LOG_FILE_NAME=log.txt
+QPIDD_FN=qpidd
+QPIDD=${CMAKE_BUILD_DIR}/src/${QPIDD_FN}
+TXTEST=${CMAKE_BUILD_DIR}/src/tests/qpid-txtest
+QPIDD_BASE_ARGS="--load-module ${STORE_MODULE} -m ${BROKER_MANAGEMENT} --auth no --default-flow-stop-threshold 0 --default-flow-resume-threshold 0 --default-queue-limit 0 --store-dir ${BASE_DIR} --log-enable ${BROKER_LOG_LEVEL} --log-to-stderr no --log-to-stdout no"
+TXTEST_INIT_STR="--init yes --transfer no --check no"
+TXTEST_RUN_STR="--init no --transfer yes --check no"
+TXTEST_CHK_STR="--init no --transfer no --check yes"
+SUCCESS_MSG="All expected messages were retrieved."
+TIMESTAMP_FORMAT="+%Y-%m-%d_%H:%M:%S"
+ANSI_RED="\e[1;31m"
+ANSI_NONE="\e[0m"
+DEFAULT_EFP_DIR=2048k
+DEFAULT_EFP_SIZE=2101248
+SIG_KILL=-9
+SIG_TERM=-15
+
+# Creates a random number into the variable named in string $1 in the range [$2..$3] (both inclusive).
+# $1: variable name as string to which random value is assigned
+# $2: minimum inclusive range of random number
+# $3: maximum inclusive range of random number
+get_random() {
+	eval $1=`python -S -c "import random; print random.randint($2,$3)"`
+}
+
+# Uses anon-uniform distribution to set a random message size.
+# Most messages must be small (0 - 1k), but we need a few medium (10k) and large (100k) ones also.
+# Sets message size into var ${MSG_SIZE}
+set_message_size() {
+	local key=0
+	get_random "key" 1 10
+	if (( "${key}" == "10" )); then  # 1 out of 10 - very large
+		get_random "MSG_SIZE" 100000 1000000
+		FILE_SIZE_MULTIPLIER=3
+	elif (( "${key}" >= "8" )); then # 2 out of 10 - large
+		get_random "MSG_SIZE" 10000 100000
+		FILE_SIZE_MULTIPLIER=2
+	elif (( "${key}" >= "6" )); then # 2 out of 10 - medium
+		get_random "MSG_SIZE" 1000 10000
+		FILE_SIZE_MULTIPLIER=1
+	else                             # 5 out of 10 - small
+		get_random "MSG_SIZE" 10 1000
+		FILE_SIZE_MULTIPLIER=1
+	fi
+}
+
+# Start or restart broker
+# $1: Log suffix: either "A" or "B". If "A", broker is started with truncation, otherwise broker is restarted with recovery.
+# $2: Truncate flag - only used if Log suffix is "A": if true, then truncate store
+# The PID of the broker is returned in ${QPIDD_PID}
+start_broker() {
+	local truncate_val
+	local truncate_str
+	if [[ "$1" == "A" ]]; then
+		if [[ $2 == true ]]; then
+			truncate_val="yes"
+			truncate_str="(Store truncated)"
+			if [[ -e ${BASE_DIR}/qls/p001/efp/${DEFAULT_EFP_DIR} ]]; then
+				for f in ${BASE_DIR}/qls/p001/efp/${DEFAULT_EFP_DIR}/*; do
+					local filesize=`stat -c%s "${f}"`
+					if (( ${filesize} != ${DEFAULT_EFP_SIZE} )); then
+						rm ${f}
+					fi
+				done
+			fi
+		else
+			truncate_val="no"
+		fi
+	else
+		truncate_val="no"
+	fi
+	echo "${QPIDD} ${QPIDD_BASE_ARGS} --truncate ${truncate_val} --log-to-file ${RESULT_DIR}/qpidd.$1.log &" > ${RESULT_DIR}/qpidd.$1.cmd
+	${QPIDD} ${QPIDD_BASE_ARGS} --truncate ${truncate_val} --log-to-file ${RESULT_DIR}/qpidd.$1.log &
+	QPIDD_PID=$!
+	echo "Broker PID=${QPIDD_PID} ${truncate_str}"  | tee -a ${LOG_FILE}
+}
+
+# Start or evaluate results of transaction test client
+# $1: Log suffix flag: either "A" or "B". If "A", client is started in test mode, otherwise client evaluates recovery.
+start_tx_test() {
+	local tx_test_params="--messages-per-tx ${MSGS_PER_TX} --tx-count 1000000 --total-messages ${TOT_MSGS} --size ${MSG_SIZE} --queues ${NUM_QUEUES}"
+	if [[ "$1" == "A" ]]; then
+		# Run in background
+		echo "${TXTEST##*/} parameters: ${tx_test_params}" | tee -a ${LOG_FILE}
+		echo "${TXTEST} ${tx_test_params} ${TXTEST_INIT_STR} &> ${RESULT_DIR}/txtest.$1.log" > ${RESULT_DIR}/txtest.$1.cmd
+		${TXTEST} ${tx_test_params} ${TXTEST_INIT_STR} &> ${RESULT_DIR}/txtest.$1.log
+		echo "${TXTEST} ${tx_test_params} ${TXTEST_RUN_STR} &> ${RESULT_DIR}/txtest.$1.log &" >> ${RESULT_DIR}/txtest.$1.cmd
+		${TXTEST} ${tx_test_params} ${TXTEST_RUN_STR} &> ${RESULT_DIR}/txtest.$1.log &
+	else
+		# Run in foreground
+		#echo "${TXTEST##*/} ${tx_test_params} ${TXTEST_CHK_STR}" | tee -a ${LOG_FILE}
+		echo "${TXTEST} ${tx_test_params} ${TXTEST_CHK_STR} &> ${RESULT_DIR}/txtest.$1.log" > ${RESULT_DIR}/txtest.$1.cmd
+		${TXTEST} ${tx_test_params} ${TXTEST_CHK_STR} &> ${RESULT_DIR}/txtest.$1.log
+	fi
+}
+
+# Search for the presence of core.* files, move them into the current result directory and run gdb against them.
+# No params
+process_core_files() {
+	ls core.* &> /dev/null
+	if (( "$?" == "0" )); then
+		for cf in core.*; do
+			gdb --batch --quiet -ex "thread apply all bt" -ex "quit" ${QPIDD} ${cf} &> ${RESULT_DIR}/${cf##*/}.gdb.txt
+			gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" ${QPIDD} ${cf} &> ${RESULT_DIR}/${cf##*/}.gdb-full.txt
+			cat ${RESULT_DIR}/${cf##*/}.gdb.txt
+			mv ${cf} ${RESULT_DIR}/
+			echo "Core file ${cf##*/} found and recovered"
+		done
+	fi
+}
+
+# Kill a process quietly
+# $1: Signal
+# $2: PID
+kill_process() {
+	kill ${1} ${2} &>> ${LOG_FILE}
+	wait ${2} &>> ${LOG_FILE}
+}
+
+# Check that test can run: No other copy of qpidd running, enough disk space
+check_ready_to_run() {
+	# Check no copy of qpidd is running
+	PID=`pgrep ${QPIDD_FN}`
+	if [[ "$?" == "0" ]]; then
+		echo "ERROR: qpidd running as pid ${PID}"
+		exit 1
+	fi
+	# Check disk is < 90% full
+	local perc_full=`df -h ${HOME} | tail -1 | awk '{print substr($5,0, length($5)-1)}'`
+	if (( ${perc_full} >= ${MAX_DISK_PERC_USED} )); then
+		echo "ERROR: Disk is too close to full (${perc_full}%)"
+		exit 2
+	fi
+}
+
+ulimit -c unlimited # Allow core files to be created
+
+RESULT_BASE_DIR_SUFFIX=`date "${TIMESTAMP_FORMAT}"`
+RESULT_BASE_DIR="${RESULT_BASE_DIR_PREFIX}.${RESULT_BASE_DIR_SUFFIX}"
+LOG_FILE=${RESULT_BASE_DIR}/${LOG_FILE_NAME}
+if [[ -n "${RESULT_BASE_DIR}" ]]; then
+	rm -rf ${RESULT_BASE_DIR}
+fi
+
+mkdir -p ${RESULT_BASE_DIR}
+for rn in `seq ${NUM_RUNS}`; do
+	# === Prepare result dir, check ready to run test, set run vars ===
+	RESULT_DIR=${RESULT_BASE_DIR}/run_${rn}
+	mkdir -p ${RESULT_DIR}
+	check_ready_to_run
+	if (( (${rn} - 1) % ${TRUNCATE_INTERVAL} == 0 )) || [[ -n ${ERROR_FLAG} ]]; then
+		TRUNCATE_FLAG=true
+	else
+		TRUNCATE_FLAG=false
+	fi
+	set_message_size
+	get_random "MSGS_PER_TX" 1 20
+	get_random "TOT_MSGS" 100 1000
+    get_random "NUM_QUEUES" 2 15
+	MIN_RUNTIME=$(( 20 * ${FILE_SIZE_MULTIPLIER} ))
+	MAX_RUNTIME=$(( 120 * ${FILE_SIZE_MULTIPLIER} ))	
+	get_random "RUN_TIME" ${MIN_RUNTIME} ${MAX_RUNTIME}
+	RECOVER_TIME=$(( ${NUM_QUEUES} * ${RECOVER_TIME_PER_QUEUE} * ${FILE_SIZE_MULTIPLIER} ))
+	echo "Run ${rn} of ${NUM_RUNS} ==============" | tee -a ${LOG_FILE}
+
+	# === PART A: Initial run of qpid-txtest ===
+	start_broker "A" ${TRUNCATE_FLAG}
+	sleep ${RECOVER_TIME} # Need a way to test if broker has started here
+	start_tx_test "A"
+	echo "Running for ${RUN_TIME} secs..." | tee -a ${LOG_FILE}
+	sleep ${RUN_TIME}
+	kill_process ${SIG_KILL} ${QPIDD_PID}
+	sleep 2
+	tar -czf ${RESULT_DIR}/qls_B.tar.gz ${RELATIVE_BASE_DIR}/qls
+
+	# === PART B: Recovery and check ===
+	start_broker "B"
+	echo "Recover time=${RECOVER_TIME} secs..." | tee -a ${LOG_FILE}
+	sleep ${RECOVER_TIME} # Need a way to test if broker has started here
+	start_tx_test "B"
+	sleep 1
+	kill_process ${SIG_TERM} ${QPIDD_PID}
+	sleep 2
+	PID=`pgrep ${QPIDD_FN}`
+	if [[ "$?" == "0" ]]; then
+		kill_process ${SIG_KILL} ${PID}
+		sleep 2
+	fi
+	tar -czf ${RESULT_DIR}/qls_C.tar.gz ${RELATIVE_BASE_DIR}/qls
+
+	# === Check for errors, cores and exceptions in logs ===
+	grep -Hn "jexception" ${RESULT_DIR}/qpidd.A.log | tee -a ${LOG_FILE}
+	grep -Hn "jexception" ${RESULT_DIR}/qpidd.B.log | tee -a ${LOG_FILE}
+	grep "${SUCCESS_MSG}" ${RESULT_DIR}/txtest.B.log &> /dev/null
+	if [[ "$?" != "0" ]]; then
+		echo "ERROR in run ${rn}" >> ${LOG_FILE}
+		echo -e "${ANSI_RED}ERROR${ANSI_NONE} in run ${rn}"
+		ERROR_FLAG=true
+	else
+		unset ERROR_FLAG
+	fi
+	sleep 2
+	process_core_files
+	echo | tee -a ${LOG_FILE}
+done
+