add an --hadrian mode to ./validate

When the '--hadrian' flag is passed to the validate script, we use hadrian
to build GHC, package it up in a binary distribution and later on run GHC's
testsuite against the said bindist, which gets installed locally in the process.

Along the way, this commit fixes a typo, an omission (build iserv binaries
before producing the bindist archive) and moves the Makefile that enables
'make install' on those bindists from being a list of strings in the code to
an actual file (it was becoming increasingly annoying to work with).

Finally, the Settings.Builders.Ghc part of this patch is necessary for being
able to use the installed binary distribution, in 'validate'.

1 files changed, 144 insertions, 57 deletions

diff --git a/validate b/validate
index 6b529cf55b..1aa7ddf4fd 100755
--- a/validate
+++ b/validate
@@ -25,6 +25,7 @@ Flags:
                     2008-07-01: 14% slower than the default.
   --quiet           More pretty build log.
                     See Note [Default build system verbosity].
+  --hadrian         Build the compiler and run the tests through hadrian.
   --help            shows this usage help.
 
   validate runs 'make -j\$THREADS', where by default THREADS is the number of
@@ -54,6 +55,7 @@ be_quiet=0
 # heavy cost of xz, which is the typical default. The options are defined in
 # mk/config.mk.in
 tar_comp=gzip
+use_hadrian=NO
 
 while [ $# -gt 0 ]
 do
@@ -82,6 +84,10 @@ do
     --quiet)
         be_quiet=1
         ;;
+    --hadrian)
+	use_hadrian=YES
+	hadrian_build_root=_validatebuild
+	;;
     --help)
         show_help
         exit 0;;
@@ -96,7 +102,12 @@ done
 check_packages () {
     if [ "$bindistdir" = "" ]
     then
-        ghc_pkg=inplace/bin/ghc-pkg
+	if [ "$use_hadrian" = "YES" ]
+	then
+	    ghc_pkg=$hadrian_build_root/stage1/bin/ghc-pkg
+	else
+            ghc_pkg=inplace/bin/ghc-pkg
+	fi
     else
         ghc_pkg="$bindistdir"/bin/ghc-pkg
     fi
@@ -127,26 +138,47 @@ fi
 
 echo "using THREADS=${threads}" >&2
 
-if type gmake > /dev/null 2> /dev/null
+if [ "$use_hadrian" = "NO" ]
 then
     make="gmake"
+   if type gmake > /dev/null 2> /dev/null
+   then
+       make="gmake"
+   else
+       make="make"
+   fi
+   if [ $be_quiet -eq 1 ]; then
+       # See Note [Default build system verbosity].
+       make="$make -s"
+   fi
+   $make -C utils/checkUniques
 else
-    make="make"
-fi
-
-if [ $be_quiet -eq 1 ]; then
-    # See Note [Default build system verbosity].
-    make="$make -s"
+    # Just build hadrian.
+    hadrian/build.sh --help > /dev/null
+    cd hadrian
+    hadrian_cmd=$(cabal new-exec -- which hadrian)
+    cd ..
+    # TODO: define a hadrian Flavour that mimics
+    # mk/flavours/validate.mk and use it here
+    # Until then, we're using the default flavour.
+    hadrian="$hadrian_cmd -j$threads --build-root=$hadrian_build_root"
+    if [ $be_quiet -eq 0 ]; then
+	hadrian="$hadrian -V"
+    fi
+    echo "Hadrian command: $hadrian"
 fi
 
-$make -C utils/checkUniques
-
 if [ $testsuite_only -eq 0 ]; then
 
 thisdir=`pwd`
 
 if [ $no_clean -eq 0 ]; then
-    $make maintainer-clean
+    if [ "$use_hadrian" = "NO" ]
+    then
+	$make maintainer-clean
+    else
+	$hadrian clean && rm -rf $hadrian_build_root
+    fi
 
     INSTDIR="$thisdir/inst"
 
@@ -154,48 +186,88 @@ if [ $no_clean -eq 0 ]; then
     ./configure --prefix="$INSTDIR" $config_args
 fi
 
-echo "Validating=YES"       >  mk/are-validating.mk
-echo "ValidateSpeed=$speed" >> mk/are-validating.mk
-echo "ValidateHpc=$hpc"     >> mk/are-validating.mk
-
-# Note [Default build system verbosity].
-#
-# From https://gitlab.haskell.org/ghc/ghc/wikis/design/build-system:
-#
-#   "The build system should clearly report what it's doing (and sometimes
-#   why), without being too verbose. It should emit actual command lines as
-#   much as possible, so that they can be inspected and cut & pasted."
-#
-# That should be the default. Only suppress commands, by setting V=0 and using
-# `make -s`, when user explicitly asks for it with `./validate --quiet`.
-if [ $be_quiet -eq 1 ]; then
-    # See Note [Default build system verbosity].
-    echo "V=0"                  >> mk/are-validating.mk # Less gunk
-fi
+if [ "$use_hadrian" = "NO" ]
+then
+    echo "Validating=YES"       >  mk/are-validating.mk
+    echo "ValidateSpeed=$speed" >> mk/are-validating.mk
+    echo "ValidateHpc=$hpc"     >> mk/are-validating.mk
+
+    # Note [Default build system verbosity].
+    #
+    # From https://gitlab.haskell.org/ghc/ghc/wikis/design/build-system:
+    #
+    #   "The build system should clearly report what it's doing (and sometimes
+    #   why), without being too verbose. It should emit actual command lines as
+    #   much as possible, so that they can be inspected and cut & pasted."
+    #
+    # That should be the default. Only suppress commands, by setting V=0 and using
+    # `make -s`, when user explicitly asks for it with `./validate --quiet`.
+    if [ $be_quiet -eq 1 ]; then
+	# See Note [Default build system verbosity].
+	echo "V=0"                  >> mk/are-validating.mk # Less gunk
+    fi
 
-$make -j$threads
-# For a "debug make", add "--debug=b --debug=m"
+    $make -j$threads
+    # For a "debug make", add "--debug=b --debug=m"
+else
+    # TODO: define a hadrian Flavour that mimics
+    # mk/flavours/validate.mk and use it here
+    $hadrian
+fi
 
 check_packages post-build
 
+bindistdir="bindisttest/install   dir"
+ghc="$bindistdir/bin/ghc"
+
 # -----------------------------------------------------------------------------
 # Build and test a binary distribution (not --fast)
 
 if [ $speed != "FAST" ]; then
-
-    $make binary-dist-prep TAR_COMP=$tar_comp
-    $make test_bindist TEST_PREP=YES TAR_COMP=$tar_comp
-
-    #
-    # Install the xhtml package into the bindist.
-    # This verifies that we can install a package into the
-    # bindist with Cabal.
-    #
-    bindistdir="bindisttest/install   dir"
+    if [ "$use_hadrian" = "NO" ]
+    then
+	$make binary-dist-prep TAR_COMP=$tar_comp
+	$make test_bindist TEST_PREP=YES TAR_COMP=$tar_comp
+    else
+	$hadrian binary-dist --docs=no-sphinx
+	cfgdir=$(find $hadrian_build_root/bindist/ -name 'configure' | head -1)
+	dir=$(dirname $cfgdir)
+	cd "$dir"
+	./configure --prefix="$thisdir/$bindistdir" && make install
+	cd $thisdir
+	"$ghc" -e 'Data.Text.IO.putStrLn (Data.Text.pack "bindist test: OK")'
+    fi
 
     check_packages post-install
 
-    $make validate_build_xhtml BINDIST_PREFIX="$thisdir/$bindistdir"
+    if [ "$use_hadrian" = "NO" ]
+    then
+	$make validate_build_xhtml BINDIST_PREFIX="$thisdir/$bindistdir"
+    else
+        cd libraries/xhtml
+        dynamicGhc=$("../../$ghc" --info | grep "GHC Dynamic" | cut -d',' -f3 | cut -d'"' -f2)
+        if [ "$dynamicGhc" = "NO" ]
+        then
+            libFlags="--enable-shared --disable-library-vanilla"
+        else
+            libFlags="--disable-shared --enable-library-vanilla"
+        fi
+        libFlags="$libFlags --disable-library-prof"
+
+       "../../$ghc" --make Setup
+       ./Setup configure \
+            --with-ghc="$thisdir/$ghc" \
+            --with-haddock="$thisdir/$bindistdir/bin/haddock" \
+            $libFlags \
+		    --global --builddir=dist-bindist \
+		    --prefix="$thisdir/$bindistdir"
+	    ./Setup build --builddir=dist-bindist
+	    ./Setup haddock -v0 --ghc-options=-optP-P --builddir=dist-bindist
+	    ./Setup install --builddir=dist-bindist
+	    ./Setup clean --builddir=dist-bindist
+	    rm -f Setup Setup.exe Setup.hi Setup.o
+	    cd ../../
+    fi
 
     check_packages post-xhtml
 fi
@@ -229,14 +301,17 @@ case "$speed" in
 SLOW)
         MAKE_TEST_TARGET=slowtest
         BINDIST="BINDIST=YES"
+	HADRIAN_TEST_SPEED=slow
         ;;
 NORMAL)
         MAKE_TEST_TARGET=test
         BINDIST="BINDIST=YES"
+	HADRIAN_TEST_SPEED=normal
         ;;
 FAST)
         MAKE_TEST_TARGET=fasttest
         BINDIST="BINDIST=NO"
+	HADRIAN_TEST_SPEED=fast
         ;;
 esac
 
@@ -252,21 +327,33 @@ fi
 
 rm -f testsuite_summary.txt testsuite_summary_stage1.txt
 
-# Use LOCAL=0, see Note [Running tests in /tmp].
-$make -C testsuite/tests $BINDIST $PYTHON_ARG \
-      $MAKE_TEST_TARGET stage=2 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
-      NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../testsuite_summary.txt \
-      JUNIT_FILE=../../testsuite.xml \
-      2>&1 | tee testlog
-
-# Run a few tests using the stage1 compiler.
-# See Note [Why is there no stage1 setup function?].
-# Don't use BINDIST=YES, as stage1 is not available in a bindist.
-$make -C testsuite/tests/stage1 $PYTHON_ARG \
-      $MAKE_TEST_TARGET stage=1 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
-      NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../../testsuite_summary_stage1.txt \
-      JUNIT_FILE=../../../testsuite_stage1.xml \
-      2>&1 | tee testlog-stage1
+if [ "$use_hadrian" = "NO" ]
+then
+    # Use LOCAL=0, see Note [Running tests in /tmp].
+    $make -C testsuite/tests $BINDIST $PYTHON_ARG \
+	  $MAKE_TEST_TARGET stage=2 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
+	  NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../testsuite_summary.txt \
+	  JUNIT_FILE=../../testsuite.xml \
+	  2>&1 | tee testlog
+
+    # Run a few tests using the stage1 compiler.
+    # See Note [Why is there no stage1 setup function?].
+    # Don't use BINDIST=YES, as stage1 is not available in a bindist.
+    $make -C testsuite/tests/stage1 $PYTHON_ARG \
+	  $MAKE_TEST_TARGET stage=1 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
+	  NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../../testsuite_summary_stage1.txt \
+	  JUNIT_FILE=../../../testsuite_stage1.xml \
+	  2>&1 | tee testlog-stage1
+else
+    testghc="$thisdir/$ghc"
+    arg="test --test-speed=$HADRIAN_TEST_SPEED \
+              --test-compiler=\"$testghc\" \
+              --summary=$thisdir/testsuite_summary.txt \
+              --summary-junit=$thisdir/testsuite.xml"
+    sh -c "$hadrian $arg"
+    # TODO: Run testsuite/tests/stage1 using the stage 1 compiler when
+    # BINDIST=NO.
+fi
 
 echo
 echo '==== STAGE 1 TESTS ==== '