From 575f1f2f9d9c38b1b93c32e9aca72e5bdf5b4826 Mon Sep 17 00:00:00 2001 From: Alina Banerjee Date: Wed, 7 Apr 2021 17:23:00 -0500 Subject: validate: add flags using Hadrian's user settings for ignoring changes in performance tests --- validate | 119 +++++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 29 deletions(-) (limited to 'validate') diff --git a/validate b/validate index d4f2ca2684..58f52438cf 100755 --- a/validate +++ b/validate @@ -11,25 +11,31 @@ https://gitlab.haskell.org/ghc/ghc/wikis/testing-patches for more information. Flags: - --no-clean don't clean first, just carry on from - a previous interrupted validation run - --testsuite-only don't build the compiler, just run the test suite - --build-only don't test the compiler, just build it - --hpc build stage2 with -fhpc, and see how much of the - compiler the test suite covers. - 2008-07-01: 63% slower than the default. - HTML generated here: testsuite/hpc_output/hpc_index.html - --fast Omit binary distribution. Omit certain tests. - --slow Build stage2 with -DDEBUG. Run tests for all WAYS, - but skip those that call compiler_stats_num_field. - 2008-07-01: 14% slower than the default. - --quiet More pretty build log. - See Note [Default build system verbosity]. - --legacy Build the compiler and run the tests through the legacy - make-based build system. - --stack Use Stack to build Hadrian and to provide the bootstrap - compiler. - --help shows this usage help. + --no-clean don't clean first, just carry on from + a previous interrupted validation run + --testsuite-only don't build the compiler, just run the test suite + --build-only don't test the compiler, just build it + --hpc build stage2 with -fhpc, and see how much of the + compiler the test suite covers. + 2008-07-01: 63% slower than the default. + HTML generated here: testsuite/hpc_output/hpc_index.html + --fast Omit binary distribution. Omit certain tests. + --slow Build stage2 with -DDEBUG. Run tests for all WAYS, + but skip those that call compiler_stats_num_field. + 2008-07-01: 14% slower than the default. + --quiet More pretty build log. + See Note [Default build system verbosity]. + --legacy Build the compiler and run the tests through the legacy + make-based build system. + --stack Use Stack to build Hadrian and to provide the bootstrap + compiler. + --ignore-perf-all Use Hadrian's user settings to ask the "runtests" driver + to ignore all changes in performance tests + --ignore-perf-incrs Use Hadrian's user settings to ask the "runtests" driver + to ignore increases in performance tests + --ignore-perf-decrs Use Hadrian's user settings to ask the "runtests" driver + to ignore decreases in performance tests + --help shows this usage help. validate runs 'make -j\$THREADS', where by default THREADS is the number of cpus your computer has +1. You can set the environment variable THREADS to @@ -60,6 +66,9 @@ be_quiet=0 tar_comp=gzip use_hadrian=YES use_stack=NO +ignore_perf_all=NO +ignore_perf_increases=NO +ignore_perf_decreases=NO hadrian_build_root=_validatebuild basedir="$($(dirname $hadrian_build_root) 2>&1 /dev/null && pwd)" @@ -101,6 +110,15 @@ do --stack) use_stack=YES ;; + --ignore-perf-all) + ignore_perf_all=YES + ;; + --ignore-perf-incrs) + ignore_perf_increases=YES + ;; + --ignore-perf-decrs) + ignore_perf_decreases=YES + ;; --help) show_help exit 0;; @@ -341,6 +359,14 @@ else PYTHON_ARG="PYTHONPATH=$(which python)" fi +if [ "$ignore_perf_all" = "YES" ]; then + test_perf_args="runtest.opts +=--ignore-perf-failures=all" +elif [ "$ignore_perf_increases" = "YES" ]; then + test_perf_args="runtest.opts +=--ignore-perf-failures=increases" +elif [ "$ignore_perf_decreases" = "YES" ]; then + test_perf_args="runtest.opts +=--ignore-perf-failures=decreases" +fi + # Run the testsuite if [ $testsuite_only -eq 1 ] || { [ $build_only -eq 0 ] && [ $testsuite_only -eq 0 ]; }; then @@ -376,7 +402,7 @@ if [ $testsuite_only -eq 1 ] || --test-compiler=\"$testghc\" \ --summary=$basedir/testsuite_summary.txt \ --summary-junit=$basedir/testsuite.xml" - sh -c "$hadrian $hadrian_args" + sh -c "$hadrian $hadrian_test_with_args \"$test_perf_args\"" # TODO: Run testsuite/tests/stage1 using the stage 1 compiler when # BINDIST=NO. @@ -399,12 +425,16 @@ if [ "$use_hadrian" = "YES" ]; then if grep '\<0 caused framework failures' testsuite_summary.txt >/dev/null 2>/dev/null && grep '\<0 unexpected passes' testsuite_summary.txt >/dev/null 2>/dev/null && - grep '\<0 unexpected failures' testsuite_summary.txt >/dev/null 2>/dev/null && - grep '\<0 unexpected stat failures' testsuite_summary.txt >/dev/null 2>/dev/null; then + grep '\<0 unexpected failures' testsuite_summary.txt >/dev/null 2>/dev/null; then no_hadrian_test_failures=1 fi -else + if + grep '\<0 unexpected stat failures' testsuite_summary.txt >/dev/null 2>/dev/null; then + + no_hadrian_perf_test_failures=1 + fi +elif [ "$use_hadrian" = "NO" ]; then if grep '\<0 caused framework failures' testsuite_summary.txt >/dev/null 2>/dev/null && grep '\<0 unexpected passes' testsuite_summary.txt >/dev/null 2>/dev/null && @@ -420,9 +450,32 @@ else fi fi -if { [ $no_hadrian_perf_test_failures = 1 ] && [ $no_hadrian_test_failures = 1 ]; } || - [ $no_make_test_failures = 1 ]; then - +if [ $no_hadrian_test_failures ] || [ $no_make_test_failures ]; then + if [ $no_hadrian_perf_test_failures -eq 1 ] || + [ $no_make_perf_test_failures -eq 1 ]; then + if [ "$ignore_perf_all" = "YES" ] || + [ "$ignore_perf_increases" = "YES" ] || + [ "$ignore_perf_decreases" = "YES" ]; then + cat <