#!/bin/bash # # libseccomp regression test automation script # # Copyright IBM Corp. 2012 # Author: Corey Bryant # # # This library is free software; you can redistribute it and/or modify it # under the terms of version 2.1 of the GNU Lesser General Public License as # published by the Free Software Foundation. # # This library is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License # for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this library; if not, see . # GLBL_ARCH_LE_SUPPORT=" \ x86 x86_64 x32 \ arm aarch64 \ loongarch64 \ mipsel mipsel64 mipsel64n32 \ ppc64le \ riscv64 \ sh" GLBL_ARCH_BE_SUPPORT=" \ m68k \ mips mips64 mips64n32 \ parisc parisc64 \ ppc ppc64 \ s390 s390x \ sheb" GLBL_ARCH_32B_SUPPORT=" \ x86 x32 \ arm \ m68k \ mips mipsel mips64n32 mipsel64n32 \ parisc \ ppc \ s390 \ sheb sh" GLBL_ARCH_64B_SUPPORT=" \ x86_64 \ aarch64 \ loongarch64 \ mips64 \ parisc64 \ ppc64 \ riscv64 \ s390x" GLBL_SYS_ARCH="../tools/scmp_arch_detect" GLBL_SYS_RESOLVER="../tools/scmp_sys_resolver" GLBL_SYS_SIM="../tools/scmp_bpf_sim" GLBL_SYS_API="../tools/scmp_api_level" #### # functions # # Dependency check # # Arguments: # 1 Dependency to check for # function check_deps() { [[ -z "$1" ]] && return which "$1" >& /dev/null return $? } # # Dependency verification # # Arguments: # 1 Dependency to check for # function verify_deps() { [[ -z "$1" ]] && return if ! check_deps "$1"; then echo "error: install \"$1\" and include it in your \$PATH" exit 1 fi } # # Print out script usage details # function usage() { cat << EOF usage: regression [-h] [-v] [-j JOBS] [-m MODE] [-a] [-b BATCH_NAME] [-l ] [-s SINGLE_TEST] [-t ] [-T ] libseccomp regression test automation script optional arguments: -h show this help message and exit -j JOBS run up to JOBS test jobs in parallel can also be set via LIBSECCOMP_TSTCFG_JOBS env variable -m MODE specified the test mode [c (default), python] can also be set via LIBSECCOMP_TSTCFG_MODE_LIST env variable -a specifies all tests are to be run -b BATCH_NAME specifies batch of tests to be run can also be set via LIBSECCOMP_TSTCFG_BATCHES env variable -l [LOG] specifies log file to write test results to -s SINGLE_TEST specifies individual test number to be run -t [TEMP_DIR] specifies directory to create temporary files in -T [TEST_TYPE] only run tests matching the specified type can also be set via LIBSECCOMP_TSTCFG_TYPE env variable -v specifies that verbose output be provided EOF } # # Match on a single word/column in a CSV string # # Arguments: # 1 string containing the CSV # 2 string containing the word to match # # Returns true/0 if a match is found false/1 otherwise. # function match_csv_word() { [[ -z $1 || -z $2 ]] && return 1 echo "$1" | sed 's/,/ /g' | grep -w "$2" } # # Generate a string representing the test number # # Arguments: # 1 string containing the batch name # 2 value of the test number from the input test data file # 3 value of the subtest number that corresponds to argument 1 # # The actual test number from the input test data file is 1 for the first # test found in the file, 2 for the second, etc. # # The subtest number is useful for batches that generate multiple tests based # on a single line of input from the test data file. The subtest number # should be set to zero if the corresponding test data is actual test data # that was read from the input file, and should be set to a value greater than # zero if the corresponding test data is generated. # function generate_test_num() { local testnumstr=$(printf '%s%%%%%03d-%05d' "$1" $2 $3) echo "$testnumstr" } # # Print the test data to the log file # # Arguments: # 1 string containing generated test number # 2 string containing line of test data # function print_data() { if [[ -n $verbose ]]; then printf "Test %s data: %s\n" "$1" "$2" >&$logfd fi } # # Print the test result to the log file # # Arguments: # 1 string containing generated test number # 2 string containing the test result (INFO, SUCCESS, ERROR, or FAILURE) # 3 string containing addition details # function print_result() { if [[ $2 == "INFO" && -z $verbose ]]; then return fi if [[ $3 == "" ]]; then printf "Test %s result: %s\n" "$1" "$2" >&$logfd else printf "Test %s result: %s %s\n" "$1" "$2" "$3" >&$logfd fi } # # Print the valgrind header to the log file # # Arguments: # 1 string containing generated test number # function print_valgrind() { if [[ -n $verbose ]]; then printf "Test %s valgrind output\n" "$1" >&$logfd fi } # # Get the low or high range value from a range specification # # Arguments: # 1 value specifying range value to retrieve: low (1) or high (2) # 2 string containing dash-separated range or a single value # function get_range() { if [[ $2 =~ ^[0-9a-fA-Fx]+-[0-9a-fA-Fx]+$ ]]; then # if there's a dash, get the low or high range value range_val=$(echo "$2" | cut -d'-' -f "$1") else # otherwise there should just be a single value range_val="$2" fi echo "$range_val" } # # Get the number sequence for a given range with increments of 1, i.e. # implement a specialized seq(1). # # We use our own implementation based on miniseq in favour to the standard seq # tool as, at least, seq of coreutils v8.23 and v8.24 has problems on 32 bit # ARM for large numbers (see the mailing thread at # https://groups.google.com/forum/#!topic/libseccomp/VtrClkXxLGA). # # Arguments: # 1 starting value # 2 last value # function get_seq() { # NOTE: this whole thing is a bit hacky, but we need to search around # for miniseq to fix 'make distcheck', someday we should fix this if [[ -x ./miniseq ]]; then ./miniseq "$1" "$2" elif [[ -x $basedir/miniseq ]]; then $basedir/miniseq "$1" "$2" else # we're often run from a subshell, so we can't simply exit echo "error: unable to find miniseq" >&2 kill $pid fi } # # Run the specified test command (with valgrind if requested) # # Arguments: # 1 string containing generated test number # 2 string containing command name # 3 string containing command options # 4 number for the stdout fd # 5 number for the stderr fd # function run_test_command() { local cmd if [[ $mode == "python" ]]; then cmd="PYTHONPATH=$PYTHONPATH" cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)" # check and adjust if we are doing a VPATH build if [[ -e "./$2.py" ]]; then cmd="$cmd /usr/bin/env python $2.py $3" else cmd="$cmd /usr/bin/env python ${srcdir}/$2.py $3" fi else cmd="$2 $3" fi # setup the stdout/stderr redirects local stdout=$4 local stderr=$5 [[ -z $stdout ]] && stdout=$logfd [[ -z $stderr ]] && stderr=$logfd # run the command eval "$cmd" 1>&$stdout 2>&$stderr # return the command's return code return $? } # # Generate pseudo-random string of alphanumeric characters # # The generated string will be no larger than the corresponding # architecture's register size. # function generate_random_data() { local rcount local rdata if [[ $arch == "x86_64" ]]; then rcount=$[ ($RANDOM % 16) + 1 ] else rcount=$[ ($RANDOM % 8) + 1 ] fi rdata=$(dd if=/dev/urandom bs=64 count=1 status=none | \ md5sum | awk '{ print $1 }' | head -c"$rcount") echo "$rdata" } # # Run the specified "bpf-sim-fuzz" test # # Tests that belong to the "bpf-sim-fuzz" test type generate a BPF filter and # then run a simulated system call test with pseudo-random fuzz data for the # syscall and argument values. Tests that belong to this test type provide the # following data on a single line in the input batch file: # # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) # StressCount - The number of fuzz tests to run against the filter # # The following test data is output to the logfile for each generated test: # # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) # Syscall - The fuzzed syscall value to be simulated against the filter # Arg0-5 - The fuzzed syscall arg values to be simulated against the filter # # Arguments: # 1 string containing the batch name # 2 value of test number from batch file # 3 string containing line of test data from batch file # function run_test_bpf_sim_fuzz() { local rc # begin splitting the test data from the line into individual variables local line=($3) local testname=${line[0]} local stress_count=${line[1]} # check for stress count configuration via environment variables [[ -n $LIBSECCOMP_TSTCFG_STRESSCNT ]] && \ stress_count=$LIBSECCOMP_TSTCFG_STRESSCNT for i in $(get_seq 1 $stress_count); do local sys=$(generate_random_data) local -a arg=($(generate_random_data) $(generate_random_data) \ $(generate_random_data) $(generate_random_data) \ $(generate_random_data) $(generate_random_data)) # get the generated sub-test num string local testnumstr=$(generate_test_num "$1" $2 $i) # set up log file test data line for this individual test, # spacing is added to align the output in the correct columns local -a COL_WIDTH=(26 17 17 17 17 17 17) local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname) testdata+=$(printf "%-${COL_WIDTH[1]}s" $sys) testdata+=$(printf "%-${COL_WIDTH[2]}s" ${arg[0]}) testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[1]}) testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[2]}) testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[3]}) testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[4]}) testdata+=$(printf "%s" ${arg[5]}) # print out the generated test data to the log file print_data "$testnumstr" "$testdata" # set up the syscall argument values to be passed to bpf_sim for i in {0..5}; do arg[$i]=" -$i ${arg[$i]} " done # run the test command and put the BPF filter in a temp file exec 4>$tmpfile run_test_command "$testnumstr" "./$testname" "-b" 4 "" rc=$? exec 4>&- if [[ $rc -ne 0 ]]; then print_result $testnumstr "ERROR" "$testname rc=$rc" stats_error=$(($stats_error+1)) return fi # simulate the fuzzed syscall data against the BPF filter, we # don't verify the resulting action since we're just testing for # stability allow=$($GLBL_SYS_SIM -f $tmpfile -s $sys \ ${arg[0]} ${arg[1]} ${arg[2]} ${arg[3]} ${arg[4]} \ ${arg[5]}) rc=$? if [[ $rc -ne 0 ]]; then print_result $testnumstr "ERROR" "bpf_sim rc=$rc" stats_error=$(($stats_error+1)) else print_result $testnumstr "SUCCESS" "" stats_success=$(($stats_success+1)) fi stats_all=$(($stats_all+1)) done } # # Run the specified "bpf-sim" test # # Tests that belong to the "bpf-sim" test type generate a BPF filter and then # run a simulated system call test to validate the filter. Tests that belong to # this test type provide the following data on a single line in the input batch # file: # # Testname - The executable test name (e.g. 01-allow, 02-basic, etc.) # Arch - The architecture that the test should be run on (all, x86, x86_64) # Syscall - The syscall to simulate against the generated filter # Arg0-5 - The syscall arguments to simulate against the generated filter # Result - The expected simulation result (ALLOW, KILL, etc.) # # If a range of syscall or argument values are specified (e.g. 1-9), a test is # generated for every combination of range values. Otherwise, the individual # test is run. # # Arguments: # 1 string containing the batch name # 2 value of test number from batch file # 3 string containing line of test data from batch file # function run_test_bpf_sim() { local rc local LOW=1 local HIGH=2 local -a arg_empty=(false false false false false false) # begin splitting the test data from the line into individual variables local line=($3) local testname=${line[0]} local testarch=${line[1]} local low_syscall #line[2] local high_syscall #line[2] local -a low_arg #line[3-8] local -a high_arg #line[3-8] local result=${line[9]} # expand the architecture list local simarch_tmp local simarch_avoid simarch_tmp="" simarch_avoid="" for arch_i in $(echo $testarch | sed -e 's/,/ /g'); do case $arch_i in all) # add the native arch simarch_tmp+=" $arch" ;; all_le) # add the native arch only if it is little endian if echo "$GLBL_ARCH_LE_SUPPORT" | grep -qw "$arch"; then simarch_tmp+=" $arch" fi ;; +all_le) # add all of the little endian architectures simarch_tmp+=" $GLBL_ARCH_LE_SUPPORT" ;; all_be) # add the native arch only if it is big endian if echo "$GLBL_ARCH_BE_SUPPORT" | grep -qw "$arch"; then simarch_tmp+=" $arch" fi ;; +all_be) # add all of the big endian architectures simarch_tmp+=" $GLBL_ARCH_BE_SUPPORT" ;; all_32) # add the native arch only if it is 32-bit if echo "$GLBL_ARCH_32B_SUPPORT" | grep -qw "$arch"; then simarch_tmp+=" $arch" fi ;; +all_32) # add all of the 32-bit architectures simarch_tmp+=" $GLBL_ARCH_32B_SUPPORT" ;; all_64) # add the native arch only if it is 64-bit if echo "$GLBL_ARCH_64B_SUPPORT" | grep -qw "$arch"; then simarch_tmp+=" $arch" fi ;; +all_64) # add all of the 64-bit architectures simarch_tmp+=" $GLBL_ARCH_64B_SUPPORT" ;; +*) # add the architecture specified simarch_tmp+=" ${arch_i:1}" ;; -*) # remove the architecture specified simarch_avoid+=" ${arch_i:1}" ;; *) # add the architecture specified if it is native if [[ "$arch_i" == "$arch" ]]; then simarch_tmp+=" $arch_i" fi ;; esac done # make sure we remove any undesired architectures local simarch_list simarch_list="" for arch_i in $simarch_tmp; do if echo "$simarch_avoid" | grep -q -v -w "$arch_i"; then simarch_list+=" $arch_i" fi done simarch_list=$(echo $simarch_list | sed -e 's/ / /g;s/^ //;') # do we have any architectures remaining in the list? if [[ $simarch_list == "" ]]; then print_result $(generate_test_num "$1" $2 1) "SKIPPED" \ "(architecture difference)" stats_skipped=$(($stats_skipped+1)) return fi # get low and high range arg values line_i=3 for arg_i in {0..5}; do low_arg[$arg_i]=$(get_range $LOW "${line[$line_i]}") high_arg[$arg_i]=$(get_range $HIGH "${line[$line_i]}") # fix up empty arg values so the nested loops work if [[ ${low_arg[$arg_i]} == "N" ]]; then arg_empty[$arg_i]=true low_arg[$arg_i]=0 high_arg[$arg_i]=0 fi line_i=$(($line_i+1)) done # loop through the selected architectures for simarch in $simarch_list; do # print architecture header if necessary if [[ $simarch != $simarch_list ]]; then echo " test arch: $simarch" >&$logfd fi # reset the subtest number local subtestnum=1 # get low and high syscall values and convert them to numbers low_syscall=$(get_range $LOW "${line[2]}") if [[ ! $low_syscall =~ ^\-?[0-9]+$ ]]; then low_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \ $low_syscall) if [[ $? -ne 0 ]]; then print_result $(generate_test_num "$1" $2 1) \ "ERROR" "sys_resolver rc=$?" stats_error=$(($stats_error+1)) return fi fi high_syscall=$(get_range $HIGH "${line[2]}") if [[ ! $high_syscall =~ ^\-?[0-9]+$ ]]; then high_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \ $high_syscall) if [[ $? -ne 0 ]]; then print_result $(generate_test_num "$1" $2 1) \ "ERROR" "sys_resolver rc=$?" stats_error=$(($stats_error+1)) return fi fi # if ranges exist, the following will loop through all syscall # and arg ranges and generate/run every combination of requested # tests; if no ranges were specified, then the single test is # run for sys in $(get_seq $low_syscall $high_syscall); do for arg0 in $(get_seq ${low_arg[0]} ${high_arg[0]}); do for arg1 in $(get_seq ${low_arg[1]} ${high_arg[1]}); do for arg2 in $(get_seq ${low_arg[2]} ${high_arg[2]}); do for arg3 in $(get_seq ${low_arg[3]} ${high_arg[3]}); do for arg4 in $(get_seq ${low_arg[4]} ${high_arg[4]}); do for arg5 in $(get_seq ${low_arg[5]} ${high_arg[5]}); do local -a arg=($arg0 $arg1 $arg2 $arg3 $arg4 $arg5) # Get the generated sub-test num string local testnumstr=$(generate_test_num "$1" $2 \ $subtestnum) # format any empty args to print to log file for i in {0..5}; do if ${arg_empty[$i]}; then arg[$i]="N" fi done # set up log file test data line for this # individual test, spacing is added to align # the output in the correct columns local -a COL_WIDTH=(26 08 14 11 17 21 09 06 06) local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname) testdata+=$(printf "%-${COL_WIDTH[1]}s" $simarch) testdata+=$(printf "%-${COL_WIDTH[2]}s" $sys) testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[0]}) testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[1]}) testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[2]}) testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[3]}) testdata+=$(printf "%-${COL_WIDTH[7]}s" ${arg[4]}) testdata+=$(printf "%-${COL_WIDTH[8]}s" ${arg[5]}) testdata+=$(printf "%-${COL_WIDTH[9]}s" $result) # print out the test data to the log file print_data "$testnumstr" "$testdata" # set up the syscall arguments to be passed to bpf_sim for i in {0..5}; do if ${arg_empty[$i]}; then arg[$i]="" else arg[$i]=" -$i ${arg[$i]} " fi done # run the test command and put the BPF in a temp file exec 4>$tmpfile run_test_command "$testnumstr" "./$testname" "-b" 4 "" rc=$? exec 4>&- if [[ $rc -ne 0 ]]; then print_result $testnumstr \ "ERROR" "$testname rc=$rc" stats_error=$(($stats_error+1)) return fi # simulate the specified syscall against the BPF filter # and verify the results action=$($GLBL_SYS_SIM -a $simarch -f $tmpfile \ -s $sys ${arg[0]} ${arg[1]} ${arg[2]} \ ${arg[3]} ${arg[4]} ${arg[5]}) rc=$? if [[ $rc -ne 0 ]]; then print_result $testnumstr \ "ERROR" "bpf_sim rc=$rc" stats_error=$(($stats_error+1)) elif [[ "$action" != "$result" ]]; then print_result $testnumstr "FAILURE" \ "bpf_sim resulted in $action" stats_failure=$(($stats_failure+1)) else print_result $testnumstr "SUCCESS" "" stats_success=$(($stats_success+1)) fi stats_all=$(($stats_all+1)) subtestnum=$(($subtestnum+1)) done # syscall done # arg0 done # arg1 done # arg2 done # arg3 done # arg4 done # arg5 done # architecture } # # Run the specified "basic" test # # Tests that belong to the "basic" test type will simply have the command # specified in the input batch file. The command must return zero for success # and non-zero for failure. # # Arguments: # 1 value of test number from batch file # 2 string containing line of test data from batch file # function run_test_basic() { local rc local cmd # if the test is a script, only run it in native/c mode if [[ $mode != "c" && "$2" == *.sh ]]; then print_result "$1" "SKIPPED" "(only valid in native/c mode)" stats_skipped=$(($stats_skipped+1)) return fi # print out the input test data to the log file print_data "$1" "$2" # check and adjust if we are doing a VPATH build if [[ -x "./$2" ]]; then cmd="./$2" else cmd="${srcdir}/$2" fi # run the command run_test_command "$1" "$cmd" "" "" "" rc=$? if [[ $rc -ne 0 ]]; then print_result $1 "FAILURE" "$2 rc=$rc" stats_failure=$(($stats_failure+1)) else print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) fi stats_all=$(($stats_all+1)) } # # Run the specified "bpf-valgrind" test # # Tests that belong to the "bpf-valgrind" test type generate a BPF filter # while running under valgrind to detect any memory errors. # # Arguments: # 1 value of test number from batch file # 2 string containing line of test data from batch file # function run_test_bpf_valgrind() { local rc # we only support the native/c test mode here if [[ $mode != "c" ]]; then print_result "$1" "SKIPPED" "(only valid in native/c mode)" stats_skipped=$(($stats_skipped+1)) return fi # print out the input test data to the log file print_data "$1" "$2" # build the command testvalgrind="valgrind \ --tool=memcheck \ --error-exitcode=1 \ --leak-check=full \ --read-var-info=yes \ --track-origins=yes \ --suppressions=$basedir/valgrind_test.supp" if [[ -n $logfile ]]; then testvalgrind+=" --log-fd=$logfd" fi if [[ -z $verbose ]]; then testvalgrind+=" --quiet --log-fd=4" fi # run the command exec 4>/dev/null print_valgrind "$1" run_test_command "$1" "$testvalgrind --" "./$2 -b" 4 2 rc=$? exec 4>&- if [[ $rc -ne 0 ]]; then print_result $1 "FAILURE" "$2 rc=$rc" stats_failure=$(($stats_failure+1)) else print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) fi stats_all=$(($stats_all+1)) } # # Run the specified "live" test # # Tests that belong to the "live" test type will attempt to run a live test # of the libseccomp library on the host system; for obvious reasons the host # system must support seccomp mode 2 for this to work correctly. # # Arguments: # 1 value of test number from batch file # 2 string containing line of test data from batch file # function run_test_live() { local rc local api local line=($2) # parse the test line line_cmd=${line[0]} line_api=${line[1]} line_act=${line[2]} line_test="$line_cmd $line_api $line_act" # check the api level api=$($GLBL_SYS_API) if [[ $api -lt $line_api ]]; then # runtime api level is too low print_result "$1" "SKIPPED" "(api level)" stats_skipped=$(($stats_skipped+1)) return fi # print out the input test data to the log file print_data "$1" "$2" # run the command exec 4>/dev/null run_test_command "$1" "./$line_cmd" "$line_act" "" 4 rc=$? exec 4>&- stats_all=$(($stats_all+1)) # setup the arch specific return values case "$arch" in x86|x86_64|x32|arm|aarch64|loongarch64|m68k|parisc|parisc64|ppc|ppc64|ppc64le|ppc|s390|s390x|riscv64|sh|sheb) rc_kill_process=159 rc_kill=159 rc_allow=160 rc_trap=161 rc_trace=162 rc_errno=163 rc_log=164 ;; mips|mipsel|mips64|mips64n32|mipsel64|mipsel64n32) rc_kill_process=140 rc_kill=140 rc_allow=160 rc_trap=161 rc_trace=162 rc_errno=163 rc_log=164 ;; *) print_result $testnumstr "ERROR" "arch $arch not supported" stats_error=$(($stats_error+1)) return ;; esac # verify the results if [[ $line_act == "KILL_PROCESS" && $rc -eq $rc_kill_process ]]; then print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) elif [[ $line_act == "KILL" && $rc -eq $rc_kill ]]; then print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) elif [[ $line_act == "ALLOW" && $rc -eq $rc_allow ]]; then print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) elif [[ $line_act == "TRAP" && $rc -eq $rc_trap ]]; then print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) elif [[ $line_act == "TRACE" ]]; then print_result $1 "ERROR" "unsupported action \"$line_act\"" stats_error=$(($stats_error+1)) elif [[ $line_act == "ERRNO" && $rc -eq $rc_errno ]]; then print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) elif [[ $line_act == "LOG" && $rc -eq $rc_log ]]; then print_result $1 "SUCCESS" "" stats_success=$(($stats_success+1)) else print_result $1 "FAILURE" "$line_test rc=$rc" stats_failure=$(($stats_failure+1)) fi } # # Run a single test from the specified batch # # Arguments: # 1 string containing the batch name # 2 value of test number from batch file # 3 string containing line of test data from batch file # 4 string containing test type that this test belongs to # function run_test() { # generate the test number string for the line of batch test data local testnumstr=$(generate_test_num "$1" $2 1) # ensure we only run tests which match the specified type match_csv_word "$type" "$4" local type_match=$? [[ -n $type && $type_match -eq 1 ]] && return # execute the function corresponding to the test type if [[ "$4" == "basic" ]]; then run_test_basic "$testnumstr" "$3" elif [[ "$4" == "bpf-sim" ]]; then run_test_bpf_sim "$1" $2 "$3" elif [[ "$4" == "bpf-sim-fuzz" ]]; then run_test_bpf_sim_fuzz "$1" $2 "$3" elif [[ "$4" == "bpf-valgrind" ]]; then # only run this test if valgrind is installed if check_deps valgrind; then run_test_bpf_valgrind "$testnumstr" "$3" else print_result $testnumstr "SKIPPED" \ "(valgrind not installed)" stats_skipped=$(($stats_skipped+1)) fi elif [[ "$4" == "live" ]]; then # only run this test if explicitly requested if [[ -n $type ]]; then run_test_live "$testnumstr" "$3" else print_result $testnumstr "SKIPPED" \ "(must specify live tests)" stats_skipped=$(($stats_skipped+1)) fi else print_result $testnumstr "ERROR" "test type $4 not supported" stats_error=$(($stats_error+1)) fi } # # Run the requested test batch # # Arguments: # 1 Batch name # function run_test_batch() { local testnum=1 local batch_name=$1 # open temporary file if [[ -n $tmpdir ]]; then tmpfile=$(mktemp -t regression_XXXXXX --tmpdir=$tmpdir) else tmpfile=$(mktemp -t regression_XXXXXX) fi # reset the stats stats_all=0 stats_skipped=0 stats_success=0 stats_failure=0 stats_error=0 # print a test batch header echo " batch name: $batch_name" >&$logfd # loop through each line and run the requested tests while read line; do # strip whitespace, comments, and blank lines line=$(echo "$line" | \ sed -e 's/^[\t ]*//;s/[\t ]*$//;' | \ sed -e '/^[#].*$/d;/^$/d') if [[ -z $line ]]; then continue fi if [[ $line =~ ^"test type": ]]; then test_type=$(echo "$line" | \ sed -e 's/^test type: //;') # print a test mode and type header echo " test mode: $mode" >&$logfd echo " test type: $test_type" >&$logfd continue fi if [[ ${single_list[@]} ]]; then for i in ${single_list[@]}; do if [ $i -eq $testnum ]; then # we're running a single test run_test "$batch_name" \ $testnum "$line" \ "$test_type" fi done else # we're running a test from a batch run_test "$batch_name" \ $testnum "$line" "$test_type" fi testnum=$(($testnum+1)) done < "$file" # dump our stats local stats=$batch_name.$mode.stats > $stats echo -n "$stats_all $stats_skipped $stats_success " >> $stats echo -n "$stats_failure $stats_error " >> $stats echo "" >> $stats # cleanup the temporary file we created rm -f $tmpfile } # # Run the requested test batch # # Arguments: # 1 Log file # 2 PID to watch # function tail_log() { local log=$1 local pid=$2 # dump the output tail -n +0 --pid=$pid -f $log # accumulate the stats local stats=$(echo $log | sed 's/\.log$/.stats/') stats_all=$(( $stats_all + $(awk '{ print $1 }' $stats) )) stats_skipped=$(( $stats_skipped + $(awk '{ print $2 }' $stats) )) stats_success=$(( $stats_success + $(awk '{ print $3 }' $stats) )) stats_failure=$(( $stats_failure + $(awk '{ print $4 }' $stats) )) stats_error=$(( $stats_error + $(awk '{ print $5 }' $stats) )) } # # Run the requested tests # function run_tests() { local job_cnt=0 local tail_cnt=0 local -a job_pids local -a job_logs # loop through all test files for file in $basedir/*.tests; do local batch_requested=false local batch_name="" # extract the batch name from the file name batch_name=$(basename $file .tests) # check if this batch was requested if [[ ${batch_list[@]} ]]; then for b in ${batch_list[@]}; do if [[ $b == $batch_name ]]; then batch_requested=true break fi done if ! $batch_requested; then continue fi fi # run the test batch run_test_batch $batch_name >& $batch_name.$mode.log & job_pids[job_cnt]=$! job_logs[job_cnt]=$batch_name.$mode.log job_cnt=$(( $job_cnt + 1 )) # output the next log if the job queue is full if [[ $(jobs | wc -l) -ge $jobs ]]; then tail_log ${job_logs[$tail_cnt]} ${job_pids[$tail_cnt]} tail_cnt=$(( $tail_cnt + 1 )) fi done # output any leftovers for i in $(seq $tail_cnt $(( $job_cnt - 1 ))); do tail_log ${job_logs[$i]} ${job_pids[$i]} done } #### # main # verify general script dependencies verify_deps head verify_deps sed verify_deps awk verify_deps tr # global variables declare -a batch_list declare -a single_list arch= batch_count=0 logfile= logfd= mode_list="" runall= singlecount=0 tmpfile="" tmpdir="" type= verbose= jobs=1 stats_all=0 stats_skipped=0 stats_success=0 stats_failure=0 stats_error=0 # set the test root directory basedir=$(dirname $0) # set the test harness pid pid=$$ # parse the command line while getopts "ab:gj:l:m:s:t:T:vh" opt; do case $opt in a) runall=1 ;; b) batch_list[batch_count]="$OPTARG" batch_count=$(($batch_count+1)) ;; j) jobs=$OPTARG ;; l) logfile="$OPTARG" ;; m) case $OPTARG in c) mode_list="$mode_list c" ;; python) verify_deps python mode_list="$mode_list python" ;; *) usage exit 1 esac ;; s) single_list[single_count]=$OPTARG single_count=$(($single_count+1)) ;; t) tmpdir="$OPTARG" ;; T) type="$OPTARG" ;; v) verbose=1 ;; h|*) usage exit 1 ;; esac done # use mode list from environment if provided [[ -z $mode_list && -n $LIBSECCOMP_TSTCFG_MODE_LIST ]] && mode_list=$LIBSECCOMP_TSTCFG_MODE_LIST # use job count from environment if provided and do some sanity checking [[ -n $LIBSECCOMP_TSTCFG_JOBS ]] && jobs=$LIBSECCOMP_TSTCFG_JOBS if [[ $jobs -lt 1 ]]; then jobs=$(cat /proc/cpuinfo | grep "^processor" | wc -l) fi # determine the mode test automatically if [[ -z $mode_list ]]; then # always perform the native c tests mode_list="c" # query the build configuration if [[ -r "../configure.h" ]]; then # python tests [[ "$(grep "ENABLE_PYTHON" ../configure.h | \ awk '{ print $3 }')" = "1" ]] && \ mode_list="$mode_list python" fi fi # check if we specified a list of tests via the environment variable if [[ -n $LIBSECCOMP_TSTCFG_BATCHES ]]; then for i in $(echo "$LIBSECCOMP_TSTCFG_BATCHES" | sed 's/,/ /g'); do batch_list[batch_count]="$i" batch_count=$(($batch_count+1)) done fi # default to all tests if batch or single tests not requested if [[ -z $batch_list ]] && [[ -z $single_list ]]; then runall=1 fi # drop any requested batch and single tests if all tests were requested if [[ -n $runall ]]; then batch_list=() single_list=() fi # check for configuration via environment variables [[ -z $type && -n $LIBSECCOMP_TSTCFG_TYPE ]] && type=$LIBSECCOMP_TSTCFG_TYPE # open log file for append (default to stdout) if [[ -n $logfile ]]; then # force single threaded to preserve the output jobs=1 logfd=3 exec 3>>"$logfile" else logfd=1 fi # determine the current system's architecture arch=$($GLBL_SYS_ARCH) # display the test output and run the requested tests echo "=============== $(date) ===============" >&$logfd echo "Regression Test Report (\"regression $*\")" >&$logfd for mode in $mode_list; do run_tests done echo "Regression Test Summary" >&$logfd echo " tests run: $stats_all" >&$logfd echo " tests skipped: $stats_skipped" >&$logfd echo " tests passed: $stats_success" >&$logfd echo " tests failed: $stats_failure" >&$logfd echo " tests errored: $stats_error" >&$logfd echo "============================================================" >&$logfd # cleanup and exit rc=0 [[ $stats_failure -gt 0 ]] && rc=$(($rc + 2)) [[ $stats_error -gt 0 ]] && rc=$(($rc + 4)) exit $rc