tests: make the regression test valgrind mode more useful

We don't really need to run valgrind multiple times for each test case,
just once should be enough.

Signed-off-by: Paul Moore <pmoore@redhat.com>

20 files changed, 184 insertions, 41 deletions

diff --git a/tests/01-sim-allow.tests b/tests/01-sim-allow.tests
index e3da13c..9630276 100644
--- a/tests/01-sim-allow.tests
+++ b/tests/01-sim-allow.tests
@@ -14,3 +14,8 @@ test type: bpf-sim-fuzz
 
 # Testname	StressCount
 01-sim-allow	50
+
+test type: bpf-valgrind
+
+# Testname
+01-sim-allow
diff --git a/tests/02-sim-basic.tests b/tests/02-sim-basic.tests
index 2121d37..07004a4 100644
--- a/tests/02-sim-basic.tests
+++ b/tests/02-sim-basic.tests
@@ -23,3 +23,8 @@ test type: bpf-sim-fuzz
 
 # Testname	StressCount
 02-sim-basic	50
+
+test type: bpf-valgrind
+
+# Testname
+02-sim-basic
diff --git a/tests/03-sim-basic_chains.tests b/tests/03-sim-basic_chains.tests
index 6692640..ef4353a 100644
--- a/tests/03-sim-basic_chains.tests
+++ b/tests/03-sim-basic_chains.tests
@@ -25,3 +25,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 03-sim-basic_chains	50
+
+test type: bpf-valgrind
+
+# Testname
+03-sim-basic_chains
diff --git a/tests/04-sim-multilevel_chains.tests b/tests/04-sim-multilevel_chains.tests
index 6f50562..cefbc4f 100644
--- a/tests/04-sim-multilevel_chains.tests
+++ b/tests/04-sim-multilevel_chains.tests
@@ -35,3 +35,8 @@ test type: bpf-sim-fuzz
 
 # Testname			StressCount
 04-sim-multilevel_chains	50
+
+test type: bpf-valgrind
+
+# Testname
+04-sim-multilevel_chains
diff --git a/tests/05-sim-long_jumps.tests b/tests/05-sim-long_jumps.tests
index bbdeaa1..03eb6d9 100644
--- a/tests/05-sim-long_jumps.tests
+++ b/tests/05-sim-long_jumps.tests
@@ -30,3 +30,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 05-sim-long_jumps	50
+
+test type: bpf-valgrind
+
+# Testname
+05-sim-long_jumps
diff --git a/tests/06-sim-actions.tests b/tests/06-sim-actions.tests
index 99d1071..f09f0a0 100644
--- a/tests/06-sim-actions.tests
+++ b/tests/06-sim-actions.tests
@@ -20,3 +20,8 @@ test type: bpf-sim-fuzz
 
 # Testname	StressCount
 06-sim-actions	50
+
+test type: bpf-valgrind
+
+# Testname
+06-sim-actions
diff --git a/tests/07-sim-db_bug_looping.tests b/tests/07-sim-db_bug_looping.tests
index f2b1b84..a7ec72b 100644
--- a/tests/07-sim-db_bug_looping.tests
+++ b/tests/07-sim-db_bug_looping.tests
@@ -16,3 +16,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 07-sim-db_bug_looping	50
+
+test type: bpf-valgrind
+
+# Testname
+07-sim-db_bug_looping
diff --git a/tests/08-sim-subtree_checks.tests b/tests/08-sim-subtree_checks.tests
index d4511bd..ed3ec42 100644
--- a/tests/08-sim-subtree_checks.tests
+++ b/tests/08-sim-subtree_checks.tests
@@ -39,3 +39,9 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 08-sim-subtree_checks	50
+
+
+test type: bpf-valgrind
+
+# Testname
+08-sim-subtree_checks
diff --git a/tests/09-sim-syscall_priority_pre.tests b/tests/09-sim-syscall_priority_pre.tests
index 3e17bc4..7b7d53f 100644
--- a/tests/09-sim-syscall_priority_pre.tests
+++ b/tests/09-sim-syscall_priority_pre.tests
@@ -19,3 +19,8 @@ test type: bpf-sim-fuzz
 
 # Testname			StressCount
 09-sim-syscall_priority_pre	50
+
+test type: bpf-valgrind
+
+# Testname
+09-sim-syscall_priority_pre
diff --git a/tests/10-sim-syscall_priority_post.tests b/tests/10-sim-syscall_priority_post.tests
index 6624d2d..aa0389f 100644
--- a/tests/10-sim-syscall_priority_post.tests
+++ b/tests/10-sim-syscall_priority_post.tests
@@ -19,3 +19,8 @@ test type: bpf-sim-fuzz
 
 # Testname			StressCount
 10-sim-syscall_priority_post	50
+
+test type: bpf-valgrind
+
+# Testname
+10-sim-syscall_priority_post
diff --git a/tests/12-sim-basic_masked_ops.tests b/tests/12-sim-basic_masked_ops.tests
index 23eada2..20e0f6d 100644
--- a/tests/12-sim-basic_masked_ops.tests
+++ b/tests/12-sim-basic_masked_ops.tests
@@ -35,3 +35,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 12-sim-basic_masked_ops	50
+
+test type: bpf-valgrind
+
+# Testname
+12-sim-basic_masked_ops
diff --git a/tests/14-sim-reset.tests b/tests/14-sim-reset.tests
index 4c42e3f..da52b2d 100644
--- a/tests/14-sim-reset.tests
+++ b/tests/14-sim-reset.tests
@@ -22,3 +22,8 @@ test type: bpf-sim-fuzz
 
 # Testname	StressCount
 14-sim-reset	50
+
+test type: bpf-valgrind
+
+# Testname
+14-sim-reset
diff --git a/tests/16-sim-arch_basic.tests b/tests/16-sim-arch_basic.tests
index 80bd705..39917b5 100644
--- a/tests/16-sim-arch_basic.tests
+++ b/tests/16-sim-arch_basic.tests
@@ -25,3 +25,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 16-sim-arch_basic	50
+
+test type: bpf-valgrind
+
+# Testname
+16-sim-arch_basic
diff --git a/tests/17-sim-arch_merge.tests b/tests/17-sim-arch_merge.tests
index c32f5fe..0f27ca9 100644
--- a/tests/17-sim-arch_merge.tests
+++ b/tests/17-sim-arch_merge.tests
@@ -22,3 +22,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 17-sim-arch_merge	50
+
+test type: bpf-valgrind
+
+# Testname
+17-sim-arch_merge
diff --git a/tests/18-sim-basic_whitelist.tests b/tests/18-sim-basic_whitelist.tests
index 2669ec4..d88d0d2 100644
--- a/tests/18-sim-basic_whitelist.tests
+++ b/tests/18-sim-basic_whitelist.tests
@@ -25,3 +25,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 18-sim-basic_whitelist	50
+
+test type: bpf-valgrind
+
+# Testname
+18-sim-basic_whitelist
diff --git a/tests/19-sim-missing_syscalls.tests b/tests/19-sim-missing_syscalls.tests
index 113fc66..5a1f244 100644
--- a/tests/19-sim-missing_syscalls.tests
+++ b/tests/19-sim-missing_syscalls.tests
@@ -14,3 +14,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 19-sim-missing_syscalls	50
+
+test type: bpf-valgrind
+
+# Testname
+19-sim-missing_syscalls
diff --git a/tests/22-sim-basic_chains_array.tests b/tests/22-sim-basic_chains_array.tests
index 6785152..b8867b7 100644
--- a/tests/22-sim-basic_chains_array.tests
+++ b/tests/22-sim-basic_chains_array.tests
@@ -24,3 +24,8 @@ test type: bpf-sim-fuzz
 
 # Testname			StressCount
 22-sim-basic_chains_array	50
+
+test type: bpf-valgrind
+
+# Testname
+22-sim-basic_chains_array
diff --git a/tests/23-sim-arch_all_basic.tests b/tests/23-sim-arch_all_basic.tests
index 02b3a79..980268f 100644
--- a/tests/23-sim-arch_all_basic.tests
+++ b/tests/23-sim-arch_all_basic.tests
@@ -21,3 +21,8 @@ test type: bpf-sim-fuzz
 
 # Testname		StressCount
 23-sim-arch_all_basic	50
+
+test type: bpf-valgrind
+
+# Testname
+23-sim-arch_all_basic
diff --git a/tests/25-sim-multilevel_chains_adv.tests b/tests/25-sim-multilevel_chains_adv.tests
index 33d3c65..1bdfa40 100644
--- a/tests/25-sim-multilevel_chains_adv.tests
+++ b/tests/25-sim-multilevel_chains_adv.tests
@@ -23,3 +23,8 @@ test type: bpf-sim-fuzz
 
 # Testname			StressCount
 25-sim-multilevel_chains_adv	50
+
+test type: bpf-valgrind
+
+# Testname
+25-sim-multilevel_chains_adv
diff --git a/tests/regression b/tests/regression
index c1d7303..bb1c33d 100755
--- a/tests/regression
+++ b/tests/regression
@@ -31,6 +31,18 @@ GLBL_SYS_SIM="../tools/scmp_bpf_sim"
 # functions
 
 #
+# Dependency check
+#
+# Arguments:
+#     1    Dependency to check for
+#
+function check_deps() {
+	[[ -z "$1" ]] && return
+	which "$1" >& /dev/null
+	return $?
+}
+
+#
 # Dependency verification
 #
 # Arguments:
@@ -38,7 +50,7 @@ GLBL_SYS_SIM="../tools/scmp_bpf_sim"
 #
 function verify_deps() {
 	[[ -z "$1" ]] && return
-	if ! which "$1" >& /dev/null; then
+	if ! check_deps "$1"; then
 		echo "error: install \"$1\" and include it in your \$PATH"
 		exit 1
 	fi
@@ -49,8 +61,8 @@ function verify_deps() {
 #
 function usage() {
 cat << EOF
-usage: regression [-h] [-m MODE] [-a] [-b BATCH_NAME] [-g] [-l <LOG>]
-                  [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>] [-v]
+usage: regression [-h] [-v] [-m MODE] [-a] [-b BATCH_NAME] [-l <LOG>]
+                  [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>]
 
 libseccomp regression test automation script
 optional arguments:
@@ -58,7 +70,6 @@ optional arguments:
   -m MODE        specified the test mode [c (default), python]
   -a             specifies all tests are to be run
   -b BATCH_NAME  specifies batch of tests to be run
-  -g             specifies that tests are to be run with valgrind
   -l [LOG]       specifies log file to write test results to
   -s SINGLE_TEST specifies individual test number to be run
   -t [TEMP_DIR]  specifies directory to create temporary files in
@@ -97,7 +108,7 @@ function generate_test_num() {
 #     2    string containing line of test data
 #
 function print_data() {
-	if $verbose; then
+	if [[ -n $verbose ]]; then
 		printf "Test %s data:     %s\n" "$1" "$2" >&$logfd
 	fi
 }
@@ -111,7 +122,7 @@ function print_data() {
 #     3    string containing addition details
 #
 function print_result() {
-	if [[ $2 == "INFO" ]] && ! $verbose; then
+	if [[ $2 == "INFO" && -z $verbose ]]; then
 		return
 	fi
 	if [[ $3 == "" ]]; then
@@ -128,8 +139,8 @@ function print_result() {
 #     1    string containing generated test number
 #
 function print_valgrind() {
-	if $verbose; then
-		printf "Test %s valgrind results:\n" "$1" >&$logfd
+	if [[ -n $verbose ]]; then
+		printf "Test %s valgrind output\n" "$1" >&$logfd
 	fi
 }
 
@@ -164,26 +175,12 @@ function get_range() {
 function run_test_command() {
 	local cmd
 
-	if $use_valgrind && $verbose; then
-		print_valgrind $1
-		if [[ $logfd -eq 3 ]]; then
-			cmd="/usr/bin/valgrind --log-fd=$logfd ./$2 $3"
-		else
-			cmd="/usr/bin/valgrind ./$2 $3"
-		fi
-	elif $use_valgrind; then
-		# with -q, valgrind will only print error messages
-		if [[ $logfd -eq 3 ]]; then
-			cmd="/usr/bin/valgrind -q --log-fd=$logfd ./$2 $3"
-		else
-			cmd="/usr/bin/valgrind -q ./$2 $3"
-		fi
-	elif [[ $mode == "python" ]]; then
+	if [[ $mode == "python" ]]; then
 		cmd="PYTHONPATH=$PYTHONPATH"
 		cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)"
-		cmd="$cmd /usr/bin/env python ./$2.py $3"
+		cmd="$cmd /usr/bin/env python $2.py $3"
 	else
-		cmd="./$2 $3"
+		cmd="$2 $3"
 	fi
 
 	# setup the stdout/stderr redirects
@@ -278,7 +275,7 @@ function run_test_bpf_sim_fuzz() {
 
 		# run the test command and put the BPF filter in a temp file
 		exec 4>$tmpfile
-		run_test_command "$testnumstr" "$testname" "-b" 4 "" ""
+		run_test_command "$testnumstr" "./$testname" "-b" 4 ""
 		rc=$?
 		exec 4>&-
 		if [[ $rc -ne 0 ]]; then
@@ -464,7 +461,7 @@ function run_test_bpf_sim() {
 
 			# run the test command and put the BPF in a temp file
 			exec 4>$tmpfile
-			run_test_command "$testnumstr" "$testname" "-b" 4 ""
+			run_test_command "$testnumstr" "./$testname" "-b" 4 ""
 			rc=$?
 			exec 4>&-
 			if [[ $rc -ne 0 ]]; then
@@ -523,8 +520,56 @@ function run_test_basic() {
 	print_data "$1" "$2"
 
 	# run the command
-	run_test_command "$1" "$2" "" "" ""
+	run_test_command "$1" "./$2" "" "" ""
+	rc=$?
+	if [[ $rc -ne 0 ]]; then
+		print_result $1 "FAILURE" "$2 rc=$rc"
+		stats_failure=$(($stats_failure+1))
+	else
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	fi
+	stats_all=$(($stats_all+1))
+}
+
+#
+# Run the specified "bpf-valgrind" test
+#
+# Tests that belong to the "bpf-valgrind" test type generate a BPF filter
+# while running under valgrind to detect any memory errors.
+#
+# Arguments:
+#     1    value of test number from batch file
+#     2    string containing line of test data from batch file
+#
+function run_test_bpf_valgrind() {
+	local rc
+	local testcmd
+
+	# print out the input test data to the log file
+	print_data "$1" "$2"
+
+	# build the command
+	testcmd="$2"
+	testvalgrind="valgrind \
+		       --tool=memcheck \
+		       --error-exitcode=1 \
+		       --leak-check=full \
+		       --read-var-info=yes \
+		       --track-origins=yes"
+	if [[ -n $logfile ]]; then
+		testvalgrind+=" --log-fd=$logfd"
+	fi
+	if [[ -z $verbose ]]; then
+		testvalgrind+=" --quiet --log-fd=4"
+	fi
+
+	# run the command
+	exec 4>/dev/null
+	print_valgrind "$1"
+	run_test_command "$1" "$testvalgrind --" "./$testcmd -b" 4 2
 	rc=$?
+	exec 4>&-
 	if [[ $rc -ne 0 ]]; then
 		print_result $1 "FAILURE" "$2 rc=$rc"
 		stats_failure=$(($stats_failure+1))
@@ -560,7 +605,7 @@ function run_test_live() {
 
 	# run the command
 	exec 4>/dev/null
-	run_test_command "$1" "$line_cmd" "$line_act" "" 4
+	run_test_command "$1" "./$line_cmd" "$line_act" "" 4
 	rc=$?
 	exec 4>&-
 
@@ -603,7 +648,7 @@ function run_test_live() {
 #
 function run_test() {
 	# generate the test number string for the line of batch test data
-	local testnumstr=$(generate_test_num "$1" $2 0)
+	local testnumstr=$(generate_test_num "$1" $2 1)
 
 	# ensure we only run tests which match the specified type
 	[[ -n $type && "$4" != "$type" ]] && return
@@ -615,6 +660,13 @@ function run_test() {
 		run_test_bpf_sim "$1" $2 "$3"
 	elif [[ "$4" == "bpf-sim-fuzz" ]]; then
 		run_test_bpf_sim_fuzz "$1" $2 "$3"
+	elif [[ "$4" == "bpf-valgrind" ]]; then
+		# only run this test if valgrind is installed
+		if check_deps valgrind; then
+			run_test_bpf_valgrind "$testnumstr" "$3"
+		else
+			stats_skipped=$(($stats_skipped+1))
+		fi
 	elif [[ "$4" == "live" ]]; then
 		# only run this test if explicitly requested
 		if [[ -n $type ]]; then
@@ -712,13 +764,12 @@ batch_count=0
 logfile=
 logfd=
 mode_list=""
-runall=false
+runall=
 singlecount=0
 tmpfile=""
 tmpdir=""
 type=
-use_valgrind=false
-verbose=false
+verbose=
 stats_all=0
 stats_skipped=0
 stats_success=0
@@ -728,16 +779,12 @@ stats_error=0
 while getopts "ab:gl:m:s:t:T:vh" opt; do
 	case $opt in
 	a)
-		runall=true
+		runall=1
 		;;
 	b)
 		batch_list[batch_count]="$OPTARG"
 		batch_count=$(($batch_count+1))
 		;;
-	g)
-		verify_deps valgrind
-		use_valgrind=true
-		;;
 	l)
 		logfile="$OPTARG"
 		;;
@@ -766,7 +813,7 @@ while getopts "ab:gl:m:s:t:T:vh" opt; do
 		type="$OPTARG"
 		;;
 	v)
-		verbose=true
+		verbose=1
 		;;
 	h|*)
 		usage
@@ -782,11 +829,11 @@ fi
 
 # default to all tests if batch or single tests not requested
 if [[ -z $batch_list ]] && [[ -z $single_list ]]; then
-	runall=true
+	runall=1
 fi
 
 # drop any requested batch and single tests if all tests were requested
-if $runall; then
+if [[ -n $runall ]]; then
 	batch_list=()
 	single_list=()
 fi