diff options
Diffstat (limited to 'libtommath/testme.sh')
| -rwxr-xr-x | libtommath/testme.sh | 226 |
1 files changed, 188 insertions, 38 deletions
diff --git a/libtommath/testme.sh b/libtommath/testme.sh index a8180bd..40fa32d 100755 --- a/libtommath/testme.sh +++ b/libtommath/testme.sh @@ -4,6 +4,7 @@ # 0 success # 128 a test failed # >0 the number of timed-out tests +# 255 parsing of parameters failed set -e @@ -21,40 +22,69 @@ _help() { echo "Usage options for $(basename $0) [--with-cc=arg [other options]]" echo - echo "Executing this script without any parameter will only run the default configuration" - echo "that has automatically been determined for the architecture you're running." + echo "Executing this script without any parameter will only run the default" + echo "configuration that has automatically been determined for the" + echo "architecture you're running." echo echo " --with-cc=* The compiler(s) to use for the tests" - echo " This is an option that will be iterated." + echo " This is an option that will be iterated." echo echo " --test-vs-mtest=* Run test vs. mtest for '*' operations." - echo " Only the first of each options will be taken into account." + echo " Only the first of each options will be" + echo " taken into account." echo - echo "To be able to specify options a compiler has to be given." - echo "All options will be tested with all MP_xBIT configurations." + echo "To be able to specify options a compiler has to be given with" + echo "the option --with-cc=compilername" + echo "All other options will be tested with all MP_xBIT configurations." echo - echo " --with-{m64,m32,mx32} The architecture(s) to build and test for," - echo " e.g. --with-mx32." - echo " This is an option that will be iterated, multiple selections are possible." - echo " The mx32 architecture is not supported by clang and will not be executed." + echo " --with-{m64,m32,mx32} The architecture(s) to build and test" + echo " for, e.g. --with-mx32." + echo " This is an option that will be iterated," + echo " multiple selections are possible." + echo " The mx32 architecture is not supported" + echo " by clang and will not be executed." echo echo " --cflags=* Give an option to the compiler," echo " e.g. --cflags=-g" - echo " This is an option that will always be passed as parameter to CC." + echo " This is an option that will always be" + echo " passed as parameter to CC." echo echo " --make-option=* Give an option to make," echo " e.g. --make-option=\"-f makefile.shared\"" - echo " This is an option that will always be passed as parameter to make." + echo " This is an option that will always be" + echo " passed as parameter to make." echo echo " --with-low-mp Also build&run tests with -DMP_{8,16,32}BIT." echo echo " --mtest-real-rand Use real random data when running mtest." echo + echo " --with-valgrind" + echo " --with-valgrind=* Run in valgrind (slow!)." + echo + echo " --with-travis-valgrind Run with valgrind on Travis on specific branches." + echo + echo " --valgrind-options Additional Valgrind options" + echo " Some of the options like e.g.:" + echo " --track-origins=yes add a lot of extra" + echo " runtime and may trigger the 30 minutes" + echo " timeout." + echo echo "Godmode:" echo - echo " --all Choose all architectures and gcc and clang as compilers" + echo " --all Choose all architectures and gcc and clang" + echo " as compilers but does not run valgrind." echo + echo " --format Runs the various source-code formatters" + echo " and generators and checks if the sources" + echo " are clean." + echo + echo " -h" echo " --help This message" + echo + echo " -v" + echo " --version Prints the version. It is just the number" + echo " of git commits to this file, no deeper" + echo " meaning attached" exit 0 } @@ -90,13 +120,49 @@ _make() _runtest() { make clean > /dev/null - _make "$1" "$2" "test_standalone" local _timeout="" which timeout >/dev/null && _timeout="timeout --foreground 90" - echo -e "\rRun test $1 $2" - $_timeout ./test > test_${suffix}.log || _die "running tests" $? + if [[ "$MAKE_OPTIONS" =~ "tune" ]] + then + # "make tune" will run "tune_it.sh" automatically, hence "autotune", but it cannot + # get switched off without some effort, so we just let it run twice for testing purposes + echo -e "\rRun autotune $1 $2" + _make "$1" "$2" "" + $_timeout $TUNE_CMD > test_${suffix}.log || _die "running autotune" $? + else + _make "$1" "$2" "test" + echo -e "\rRun test $1 $2" + $_timeout ./test > test_${suffix}.log || _die "running tests" $? + fi +} + +# This is not much more of a C&P of _runtest with a different timeout +# and the additional valgrind call. +# TODO: merge +_runvalgrind() +{ + make clean > /dev/null + local _timeout="" + # 30 minutes? Yes. Had it at 20 minutes and the Valgrind run needed over 25 minutes. + # A bit too close for comfort. + which timeout >/dev/null && _timeout="timeout --foreground 1800" +echo "MAKE_OPTIONS = \"$MAKE_OPTIONS\"" + if [[ "$MAKE_OPTIONS" =~ "tune" ]] + then +echo "autotune branch" + _make "$1" "$2" "" + # The shell used for /bin/sh is DASH 0.5.7-4ubuntu1 on the author's machine which fails valgrind, so + # we just run on instance of etc/tune with the same options as in etc/tune_it.sh + echo -e "\rRun etc/tune $1 $2 once inside valgrind" + $_timeout $VALGRIND_BIN $VALGRIND_OPTS $TUNE_CMD > test_${suffix}.log || _die "running etc/tune" $? + else + _make "$1" "$2" "test" + echo -e "\rRun test $1 $2 inside valgrind" + $_timeout $VALGRIND_BIN $VALGRIND_OPTS ./test > test_${suffix}.log || _die "running tests" $? + fi } + _banner() { echo "uname="$(uname -a) @@ -121,6 +187,29 @@ CFLAGS="" WITH_LOW_MP="" TEST_VS_MTEST="" MTEST_RAND="" +# timed with an AMD A8-6600K +# 25 minutes +#VALGRIND_OPTS=" --track-origins=yes --leak-check=full --show-leak-kinds=all --error-exitcode=1 " +# 9 minutes (14 minutes with --test-vs-mtest=333333 --mtest-real-rand) +VALGRIND_OPTS=" --leak-check=full --show-leak-kinds=all --error-exitcode=1 " +#VALGRIND_OPTS="" +VALGRIND_BIN="" +CHECK_FORMAT="" +TUNE_CMD="./etc/tune -t -r 10 -L 3" + +alive_pid=0 + +function kill_alive() { + disown $alive_pid || true + kill $alive_pid 2>/dev/null +} + +function start_alive_printing() { + [ "$alive_pid" == "0" ] || return 0; + for i in `seq 1 10` ; do sleep 300 && echo "Tests still in Progress..."; done & + alive_pid=$! + trap kill_alive EXIT +} while [ $# -gt 0 ]; do @@ -134,6 +223,30 @@ do --cflags=*) CFLAGS="$CFLAGS ${1#*=}" ;; + --valgrind-options=*) + VALGRIND_OPTS="$VALGRIND_OPTS ${1#*=}" + ;; + --with-valgrind*) + if [[ ${1#*d} != "" ]] + then + VALGRIND_BIN="${1#*=}" + else + VALGRIND_BIN="valgrind" + fi + start_alive_printing + ;; + --with-travis-valgrind*) + if [[ ("$TRAVIS_BRANCH" == "develop" && "$TRAVIS_PULL_REQUEST" == "false") || "$TRAVIS_BRANCH" == *"valgrind"* || "$TRAVIS_COMMIT_MESSAGE" == *"valgrind"* ]] + then + if [[ ${1#*d} != "" ]] + then + VALGRIND_BIN="${1#*=}" + else + VALGRIND_BIN="valgrind" + fi + start_alive_printing + fi + ;; --make-option=*) MAKE_OPTIONS="$MAKE_OPTIONS ${1#*=}" ;; @@ -145,12 +258,16 @@ do if ! [ "$TEST_VS_MTEST" -eq "$TEST_VS_MTEST" ] 2> /dev/null then echo "--test-vs-mtest Parameter has to be int" - exit -1 + exit 255 fi + start_alive_printing ;; --mtest-real-rand) MTEST_RAND="-DLTM_MTEST_REAL_RAND" ;; + --format) + CHECK_FORMAT="1" + ;; --all) COMPILERS="gcc clang" ARCHFLAGS="-m64 -m32 -mx32" @@ -158,6 +275,10 @@ do --help | -h) _help ;; + --version | -v) + echo $(git rev-list HEAD --count -- testme.sh) || echo "Unknown. Please run in original libtommath git repository." + exit 0 + ;; *) echo "Ignoring option ${1}" ;; @@ -165,18 +286,42 @@ do shift done -# default to gcc if no compiler is defined but some other options +function _check_git() { + git update-index --refresh >/dev/null || true + git diff-index --quiet HEAD -- . || ( echo "FAILURE: $*" && exit 1 ) +} + +if [[ "$CHECK_FORMAT" == "1" ]] +then + make astyle + _check_git "make astyle" + perl helper.pl --update-files + _check_git "helper.pl --update-files" + perl helper.pl --check-all + _check_git "helper.pl --check-all" + exit $? +fi + +[[ "$VALGRIND_BIN" == "" ]] && VALGRIND_OPTS="" + +# default to CC environment variable if no compiler is defined but some other options if [[ "$COMPILERS" == "" ]] && [[ "$ARCHFLAGS$MAKE_OPTIONS$CFLAGS" != "" ]] then - COMPILERS="gcc" -# default to gcc and run only default config if no option is given + COMPILERS="$CC" +# default to CC environment variable and run only default config if no option is given elif [[ "$COMPILERS" == "" ]] then - _banner gcc - _runtest "gcc" "" + _banner "$CC" + if [[ "$VALGRIND_BIN" != "" ]] + then + _runvalgrind "$CC" "" + else + _runtest "$CC" "" + fi _exit fi + archflags=( $ARCHFLAGS ) compilers=( $COMPILERS ) @@ -191,21 +336,18 @@ _banner if [[ "$TEST_VS_MTEST" != "" ]] then make clean > /dev/null - _make "${compilers[0]} ${archflags[0]}" "$CFLAGS" "test" + _make "${compilers[0]} ${archflags[0]}" "$CFLAGS" "mtest_opponent" echo _make "gcc" "$MTEST_RAND" "mtest" echo echo "Run test vs. mtest for $TEST_VS_MTEST iterations" - for i in `seq 1 10` ; do sleep 500 && echo alive; done & - alive_pid=$! _timeout="" - which timeout >/dev/null && _timeout="timeout --foreground 900" - $_TIMEOUT ./mtest/mtest $TEST_VS_MTEST | ./test > test.log - disown $alive_pid - kill $alive_pid 2>/dev/null - head -n 5 test.log - tail -n 2 test.log - exit 0 + which timeout >/dev/null && _timeout="timeout --foreground 1800" + $_timeout ./mtest/mtest $TEST_VS_MTEST | $VALGRIND_BIN $VALGRIND_OPTS ./mtest_opponent > valgrind_test.log 2> test_vs_mtest_err.log + retval=$? + head -n 5 valgrind_test.log + tail -n 2 valgrind_test.log + exit $retval fi for i in "${compilers[@]}" @@ -232,12 +374,20 @@ do echo "clang -mx32 tests skipped" continue fi - - _runtest "$i $a" "$CFLAGS" - [ "$WITH_LOW_MP" != "1" ] && continue - _runtest "$i $a" "-DMP_8BIT $CFLAGS" - _runtest "$i $a" "-DMP_16BIT $CFLAGS" - _runtest "$i $a" "-DMP_32BIT $CFLAGS" + if [[ "$VALGRIND_BIN" != "" ]] + then + _runvalgrind "$i $a" "$CFLAGS" + [ "$WITH_LOW_MP" != "1" ] && continue + _runvalgrind "$i $a" "-DMP_8BIT $CFLAGS" + _runvalgrind "$i $a" "-DMP_16BIT $CFLAGS" + _runvalgrind "$i $a" "-DMP_32BIT $CFLAGS" + else + _runtest "$i $a" "$CFLAGS" + [ "$WITH_LOW_MP" != "1" ] && continue + _runtest "$i $a" "-DMP_8BIT $CFLAGS" + _runtest "$i $a" "-DMP_16BIT $CFLAGS" + _runtest "$i $a" "-DMP_32BIT $CFLAGS" + fi done done |