diff options
-rw-r--r-- | tests/01-sim-allow.tests | 5 | ||||
-rw-r--r-- | tests/02-sim-basic.tests | 5 | ||||
-rw-r--r-- | tests/03-sim-basic_chains.tests | 5 | ||||
-rw-r--r-- | tests/04-sim-multilevel_chains.tests | 5 | ||||
-rw-r--r-- | tests/05-sim-long_jumps.tests | 5 | ||||
-rw-r--r-- | tests/06-sim-actions.tests | 5 | ||||
-rw-r--r-- | tests/07-sim-db_bug_looping.tests | 5 | ||||
-rw-r--r-- | tests/08-sim-subtree_checks.tests | 6 | ||||
-rw-r--r-- | tests/09-sim-syscall_priority_pre.tests | 5 | ||||
-rw-r--r-- | tests/10-sim-syscall_priority_post.tests | 5 | ||||
-rw-r--r-- | tests/12-sim-basic_masked_ops.tests | 5 | ||||
-rw-r--r-- | tests/14-sim-reset.tests | 5 | ||||
-rw-r--r-- | tests/16-sim-arch_basic.tests | 5 | ||||
-rw-r--r-- | tests/17-sim-arch_merge.tests | 5 | ||||
-rw-r--r-- | tests/18-sim-basic_whitelist.tests | 5 | ||||
-rw-r--r-- | tests/19-sim-missing_syscalls.tests | 5 | ||||
-rw-r--r-- | tests/22-sim-basic_chains_array.tests | 5 | ||||
-rw-r--r-- | tests/23-sim-arch_all_basic.tests | 5 | ||||
-rw-r--r-- | tests/25-sim-multilevel_chains_adv.tests | 5 | ||||
-rwxr-xr-x | tests/regression | 129 |
20 files changed, 184 insertions, 41 deletions
diff --git a/tests/01-sim-allow.tests b/tests/01-sim-allow.tests index e3da13c..9630276 100644 --- a/tests/01-sim-allow.tests +++ b/tests/01-sim-allow.tests @@ -14,3 +14,8 @@ test type: bpf-sim-fuzz # Testname StressCount 01-sim-allow 50 + +test type: bpf-valgrind + +# Testname +01-sim-allow diff --git a/tests/02-sim-basic.tests b/tests/02-sim-basic.tests index 2121d37..07004a4 100644 --- a/tests/02-sim-basic.tests +++ b/tests/02-sim-basic.tests @@ -23,3 +23,8 @@ test type: bpf-sim-fuzz # Testname StressCount 02-sim-basic 50 + +test type: bpf-valgrind + +# Testname +02-sim-basic diff --git a/tests/03-sim-basic_chains.tests b/tests/03-sim-basic_chains.tests index 6692640..ef4353a 100644 --- a/tests/03-sim-basic_chains.tests +++ b/tests/03-sim-basic_chains.tests @@ -25,3 +25,8 @@ test type: bpf-sim-fuzz # Testname StressCount 03-sim-basic_chains 50 + +test type: bpf-valgrind + +# Testname +03-sim-basic_chains diff --git a/tests/04-sim-multilevel_chains.tests b/tests/04-sim-multilevel_chains.tests index 6f50562..cefbc4f 100644 --- a/tests/04-sim-multilevel_chains.tests +++ b/tests/04-sim-multilevel_chains.tests @@ -35,3 +35,8 @@ test type: bpf-sim-fuzz # Testname StressCount 04-sim-multilevel_chains 50 + +test type: bpf-valgrind + +# Testname +04-sim-multilevel_chains diff --git a/tests/05-sim-long_jumps.tests b/tests/05-sim-long_jumps.tests index bbdeaa1..03eb6d9 100644 --- a/tests/05-sim-long_jumps.tests +++ b/tests/05-sim-long_jumps.tests @@ -30,3 +30,8 @@ test type: bpf-sim-fuzz # Testname StressCount 05-sim-long_jumps 50 + +test type: bpf-valgrind + +# Testname +05-sim-long_jumps diff --git a/tests/06-sim-actions.tests b/tests/06-sim-actions.tests index 99d1071..f09f0a0 100644 --- a/tests/06-sim-actions.tests +++ b/tests/06-sim-actions.tests @@ -20,3 +20,8 @@ test type: bpf-sim-fuzz # Testname StressCount 06-sim-actions 50 + +test type: bpf-valgrind + +# Testname +06-sim-actions diff --git a/tests/07-sim-db_bug_looping.tests b/tests/07-sim-db_bug_looping.tests index f2b1b84..a7ec72b 100644 --- a/tests/07-sim-db_bug_looping.tests +++ b/tests/07-sim-db_bug_looping.tests @@ -16,3 +16,8 @@ test type: bpf-sim-fuzz # Testname StressCount 07-sim-db_bug_looping 50 + +test type: bpf-valgrind + +# Testname +07-sim-db_bug_looping diff --git a/tests/08-sim-subtree_checks.tests b/tests/08-sim-subtree_checks.tests index d4511bd..ed3ec42 100644 --- a/tests/08-sim-subtree_checks.tests +++ b/tests/08-sim-subtree_checks.tests @@ -39,3 +39,9 @@ test type: bpf-sim-fuzz # Testname StressCount 08-sim-subtree_checks 50 + + +test type: bpf-valgrind + +# Testname +08-sim-subtree_checks diff --git a/tests/09-sim-syscall_priority_pre.tests b/tests/09-sim-syscall_priority_pre.tests index 3e17bc4..7b7d53f 100644 --- a/tests/09-sim-syscall_priority_pre.tests +++ b/tests/09-sim-syscall_priority_pre.tests @@ -19,3 +19,8 @@ test type: bpf-sim-fuzz # Testname StressCount 09-sim-syscall_priority_pre 50 + +test type: bpf-valgrind + +# Testname +09-sim-syscall_priority_pre diff --git a/tests/10-sim-syscall_priority_post.tests b/tests/10-sim-syscall_priority_post.tests index 6624d2d..aa0389f 100644 --- a/tests/10-sim-syscall_priority_post.tests +++ b/tests/10-sim-syscall_priority_post.tests @@ -19,3 +19,8 @@ test type: bpf-sim-fuzz # Testname StressCount 10-sim-syscall_priority_post 50 + +test type: bpf-valgrind + +# Testname +10-sim-syscall_priority_post diff --git a/tests/12-sim-basic_masked_ops.tests b/tests/12-sim-basic_masked_ops.tests index 23eada2..20e0f6d 100644 --- a/tests/12-sim-basic_masked_ops.tests +++ b/tests/12-sim-basic_masked_ops.tests @@ -35,3 +35,8 @@ test type: bpf-sim-fuzz # Testname StressCount 12-sim-basic_masked_ops 50 + +test type: bpf-valgrind + +# Testname +12-sim-basic_masked_ops diff --git a/tests/14-sim-reset.tests b/tests/14-sim-reset.tests index 4c42e3f..da52b2d 100644 --- a/tests/14-sim-reset.tests +++ b/tests/14-sim-reset.tests @@ -22,3 +22,8 @@ test type: bpf-sim-fuzz # Testname StressCount 14-sim-reset 50 + +test type: bpf-valgrind + +# Testname +14-sim-reset diff --git a/tests/16-sim-arch_basic.tests b/tests/16-sim-arch_basic.tests index 80bd705..39917b5 100644 --- a/tests/16-sim-arch_basic.tests +++ b/tests/16-sim-arch_basic.tests @@ -25,3 +25,8 @@ test type: bpf-sim-fuzz # Testname StressCount 16-sim-arch_basic 50 + +test type: bpf-valgrind + +# Testname +16-sim-arch_basic diff --git a/tests/17-sim-arch_merge.tests b/tests/17-sim-arch_merge.tests index c32f5fe..0f27ca9 100644 --- a/tests/17-sim-arch_merge.tests +++ b/tests/17-sim-arch_merge.tests @@ -22,3 +22,8 @@ test type: bpf-sim-fuzz # Testname StressCount 17-sim-arch_merge 50 + +test type: bpf-valgrind + +# Testname +17-sim-arch_merge diff --git a/tests/18-sim-basic_whitelist.tests b/tests/18-sim-basic_whitelist.tests index 2669ec4..d88d0d2 100644 --- a/tests/18-sim-basic_whitelist.tests +++ b/tests/18-sim-basic_whitelist.tests @@ -25,3 +25,8 @@ test type: bpf-sim-fuzz # Testname StressCount 18-sim-basic_whitelist 50 + +test type: bpf-valgrind + +# Testname +18-sim-basic_whitelist diff --git a/tests/19-sim-missing_syscalls.tests b/tests/19-sim-missing_syscalls.tests index 113fc66..5a1f244 100644 --- a/tests/19-sim-missing_syscalls.tests +++ b/tests/19-sim-missing_syscalls.tests @@ -14,3 +14,8 @@ test type: bpf-sim-fuzz # Testname StressCount 19-sim-missing_syscalls 50 + +test type: bpf-valgrind + +# Testname +19-sim-missing_syscalls diff --git a/tests/22-sim-basic_chains_array.tests b/tests/22-sim-basic_chains_array.tests index 6785152..b8867b7 100644 --- a/tests/22-sim-basic_chains_array.tests +++ b/tests/22-sim-basic_chains_array.tests @@ -24,3 +24,8 @@ test type: bpf-sim-fuzz # Testname StressCount 22-sim-basic_chains_array 50 + +test type: bpf-valgrind + +# Testname +22-sim-basic_chains_array diff --git a/tests/23-sim-arch_all_basic.tests b/tests/23-sim-arch_all_basic.tests index 02b3a79..980268f 100644 --- a/tests/23-sim-arch_all_basic.tests +++ b/tests/23-sim-arch_all_basic.tests @@ -21,3 +21,8 @@ test type: bpf-sim-fuzz # Testname StressCount 23-sim-arch_all_basic 50 + +test type: bpf-valgrind + +# Testname +23-sim-arch_all_basic diff --git a/tests/25-sim-multilevel_chains_adv.tests b/tests/25-sim-multilevel_chains_adv.tests index 33d3c65..1bdfa40 100644 --- a/tests/25-sim-multilevel_chains_adv.tests +++ b/tests/25-sim-multilevel_chains_adv.tests @@ -23,3 +23,8 @@ test type: bpf-sim-fuzz # Testname StressCount 25-sim-multilevel_chains_adv 50 + +test type: bpf-valgrind + +# Testname +25-sim-multilevel_chains_adv diff --git a/tests/regression b/tests/regression index c1d7303..bb1c33d 100755 --- a/tests/regression +++ b/tests/regression @@ -31,6 +31,18 @@ GLBL_SYS_SIM="../tools/scmp_bpf_sim" # functions # +# Dependency check +# +# Arguments: +# 1 Dependency to check for +# +function check_deps() { + [[ -z "$1" ]] && return + which "$1" >& /dev/null + return $? +} + +# # Dependency verification # # Arguments: @@ -38,7 +50,7 @@ GLBL_SYS_SIM="../tools/scmp_bpf_sim" # function verify_deps() { [[ -z "$1" ]] && return - if ! which "$1" >& /dev/null; then + if ! check_deps "$1"; then echo "error: install \"$1\" and include it in your \$PATH" exit 1 fi @@ -49,8 +61,8 @@ function verify_deps() { # function usage() { cat << EOF -usage: regression [-h] [-m MODE] [-a] [-b BATCH_NAME] [-g] [-l <LOG>] - [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>] [-v] +usage: regression [-h] [-v] [-m MODE] [-a] [-b BATCH_NAME] [-l <LOG>] + [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>] libseccomp regression test automation script optional arguments: @@ -58,7 +70,6 @@ optional arguments: -m MODE specified the test mode [c (default), python] -a specifies all tests are to be run -b BATCH_NAME specifies batch of tests to be run - -g specifies that tests are to be run with valgrind -l [LOG] specifies log file to write test results to -s SINGLE_TEST specifies individual test number to be run -t [TEMP_DIR] specifies directory to create temporary files in @@ -97,7 +108,7 @@ function generate_test_num() { # 2 string containing line of test data # function print_data() { - if $verbose; then + if [[ -n $verbose ]]; then printf "Test %s data: %s\n" "$1" "$2" >&$logfd fi } @@ -111,7 +122,7 @@ function print_data() { # 3 string containing addition details # function print_result() { - if [[ $2 == "INFO" ]] && ! $verbose; then + if [[ $2 == "INFO" && -z $verbose ]]; then return fi if [[ $3 == "" ]]; then @@ -128,8 +139,8 @@ function print_result() { # 1 string containing generated test number # function print_valgrind() { - if $verbose; then - printf "Test %s valgrind results:\n" "$1" >&$logfd + if [[ -n $verbose ]]; then + printf "Test %s valgrind output\n" "$1" >&$logfd fi } @@ -164,26 +175,12 @@ function get_range() { function run_test_command() { local cmd - if $use_valgrind && $verbose; then - print_valgrind $1 - if [[ $logfd -eq 3 ]]; then - cmd="/usr/bin/valgrind --log-fd=$logfd ./$2 $3" - else - cmd="/usr/bin/valgrind ./$2 $3" - fi - elif $use_valgrind; then - # with -q, valgrind will only print error messages - if [[ $logfd -eq 3 ]]; then - cmd="/usr/bin/valgrind -q --log-fd=$logfd ./$2 $3" - else - cmd="/usr/bin/valgrind -q ./$2 $3" - fi - elif [[ $mode == "python" ]]; then + if [[ $mode == "python" ]]; then cmd="PYTHONPATH=$PYTHONPATH" cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)" - cmd="$cmd /usr/bin/env python ./$2.py $3" + cmd="$cmd /usr/bin/env python $2.py $3" else - cmd="./$2 $3" + cmd="$2 $3" fi # setup the stdout/stderr redirects @@ -278,7 +275,7 @@ function run_test_bpf_sim_fuzz() { # run the test command and put the BPF filter in a temp file exec 4>$tmpfile - run_test_command "$testnumstr" "$testname" "-b" 4 "" "" + run_test_command "$testnumstr" "./$testname" "-b" 4 "" rc=$? exec 4>&- if [[ $rc -ne 0 ]]; then @@ -464,7 +461,7 @@ function run_test_bpf_sim() { # run the test command and put the BPF in a temp file exec 4>$tmpfile - run_test_command "$testnumstr" "$testname" "-b" 4 "" + run_test_command "$testnumstr" "./$testname" "-b" 4 "" rc=$? exec 4>&- if [[ $rc -ne 0 ]]; then @@ -523,8 +520,56 @@ function run_test_basic() { print_data "$1" "$2" # run the command - run_test_command "$1" "$2" "" "" "" + run_test_command "$1" "./$2" "" "" "" + rc=$? + if [[ $rc -ne 0 ]]; then + print_result $1 "FAILURE" "$2 rc=$rc" + stats_failure=$(($stats_failure+1)) + else + print_result $1 "SUCCESS" "" + stats_success=$(($stats_success+1)) + fi + stats_all=$(($stats_all+1)) +} + +# +# Run the specified "bpf-valgrind" test +# +# Tests that belong to the "bpf-valgrind" test type generate a BPF filter +# while running under valgrind to detect any memory errors. +# +# Arguments: +# 1 value of test number from batch file +# 2 string containing line of test data from batch file +# +function run_test_bpf_valgrind() { + local rc + local testcmd + + # print out the input test data to the log file + print_data "$1" "$2" + + # build the command + testcmd="$2" + testvalgrind="valgrind \ + --tool=memcheck \ + --error-exitcode=1 \ + --leak-check=full \ + --read-var-info=yes \ + --track-origins=yes" + if [[ -n $logfile ]]; then + testvalgrind+=" --log-fd=$logfd" + fi + if [[ -z $verbose ]]; then + testvalgrind+=" --quiet --log-fd=4" + fi + + # run the command + exec 4>/dev/null + print_valgrind "$1" + run_test_command "$1" "$testvalgrind --" "./$testcmd -b" 4 2 rc=$? + exec 4>&- if [[ $rc -ne 0 ]]; then print_result $1 "FAILURE" "$2 rc=$rc" stats_failure=$(($stats_failure+1)) @@ -560,7 +605,7 @@ function run_test_live() { # run the command exec 4>/dev/null - run_test_command "$1" "$line_cmd" "$line_act" "" 4 + run_test_command "$1" "./$line_cmd" "$line_act" "" 4 rc=$? exec 4>&- @@ -603,7 +648,7 @@ function run_test_live() { # function run_test() { # generate the test number string for the line of batch test data - local testnumstr=$(generate_test_num "$1" $2 0) + local testnumstr=$(generate_test_num "$1" $2 1) # ensure we only run tests which match the specified type [[ -n $type && "$4" != "$type" ]] && return @@ -615,6 +660,13 @@ function run_test() { run_test_bpf_sim "$1" $2 "$3" elif [[ "$4" == "bpf-sim-fuzz" ]]; then run_test_bpf_sim_fuzz "$1" $2 "$3" + elif [[ "$4" == "bpf-valgrind" ]]; then + # only run this test if valgrind is installed + if check_deps valgrind; then + run_test_bpf_valgrind "$testnumstr" "$3" + else + stats_skipped=$(($stats_skipped+1)) + fi elif [[ "$4" == "live" ]]; then # only run this test if explicitly requested if [[ -n $type ]]; then @@ -712,13 +764,12 @@ batch_count=0 logfile= logfd= mode_list="" -runall=false +runall= singlecount=0 tmpfile="" tmpdir="" type= -use_valgrind=false -verbose=false +verbose= stats_all=0 stats_skipped=0 stats_success=0 @@ -728,16 +779,12 @@ stats_error=0 while getopts "ab:gl:m:s:t:T:vh" opt; do case $opt in a) - runall=true + runall=1 ;; b) batch_list[batch_count]="$OPTARG" batch_count=$(($batch_count+1)) ;; - g) - verify_deps valgrind - use_valgrind=true - ;; l) logfile="$OPTARG" ;; @@ -766,7 +813,7 @@ while getopts "ab:gl:m:s:t:T:vh" opt; do type="$OPTARG" ;; v) - verbose=true + verbose=1 ;; h|*) usage @@ -782,11 +829,11 @@ fi # default to all tests if batch or single tests not requested if [[ -z $batch_list ]] && [[ -z $single_list ]]; then - runall=true + runall=1 fi # drop any requested batch and single tests if all tests were requested -if $runall; then +if [[ -n $runall ]]; then batch_list=() single_list=() fi |