gitlab-ci: Consolidate CI logic

This moves nearly all of the CI logic to .gitlab/ci.sh. This improves
things in a number of ways:

 * it's harder for inconsistencies to arise between architectures
 * it's easier to share logic between architectures
 * on Windows, it's easier to ensure that all CI steps are executed from
   within a properly initialized mingw session.

While in town I also add a FreeBSD build job and update the Windows job
to use the gitlab-runner PowerShell executor, since cmd.exe will be
deprecated soon (fixing #17699).

5 files changed, 575 insertions, 336 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 292ff7db96..990fa11f91 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,17 +5,16 @@ variables:
   DOCKER_REV: 408eff66aef6ca2b44446c694c5a56d6ca0460cc
 
   # Sequential version number capturing the versions of all tools fetched by
-  # .gitlab/win32-init.sh.
+  # .gitlab/ci.sh.
   WINDOWS_TOOLCHAIN_VERSION: 1
 
   # Disable shallow clones; they break our linting rules
   GIT_DEPTH: 0
 
-before_script:
-  - git submodule sync --recursive
-  - git submodule update --init --recursive
-  - git checkout .gitmodules
-  - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
+  # Overridden by individual jobs
+  CONFIGURE_ARGS: ""
+
+  GIT_SUBMODULE_STRATEGY: "recursive"
 
 stages:
   - lint        # Source linting
@@ -37,6 +36,9 @@ stages:
     - web
 
 .release: &release
+  variables:
+    BUILD_FLAVOUR: "perf"
+    FLAVOUR: "perf"
   artifacts:
     when: always
     expire_in: 1 year
@@ -194,26 +196,10 @@ lint-release-changelogs:
   variables:
     FLAVOUR: "validate"
   script:
-    - cabal update
-    - git clean -xdf && git submodule foreach git clean -xdf
-    - .gitlab/prepare-system.sh
-    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
-    - ./boot
-    - ./configure $CONFIGURE_ARGS
-    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
-    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
-    - export TOP=$(pwd)
-    - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../
-    - |
-      # Prepare to push git notes.
-      export METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
-      git config user.email "ben+ghc-ci@smart-cactus.org"
-      git config user.name "GHC GitLab CI"
-      .gitlab/test-metrics.sh pull
-    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc || (.gitlab/test-metrics.sh push && false)
-    - |
-      # Push git notes.
-      .gitlab/test-metrics.sh push
+    - .gitlab/ci.sh setup
+    - .gitlab/ci.sh configure
+    - .gitlab/ci.sh build_hadrian
+    - .gitlab/ci.sh test_hadrian
   cache:
     key: hadrian
     paths:
@@ -238,6 +224,8 @@ lint-release-changelogs:
     - git submodule update --init --recursive
     - git checkout .gitmodules
     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
+  after_script:
+    - .gitlab/ci.sh clean
   tags:
     - x86_64-linux
 
@@ -270,7 +258,7 @@ hadrian-ghc-in-ghci:
     - cabal update
     - cd hadrian; cabal new-build --project-file=ci.project; cd ..
     - git clean -xdf && git submodule foreach git clean -xdf
-    - .gitlab/prepare-system.sh
+    - .gitlab/ci.sh setup
     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
     - ./boot
     - ./configure $CONFIGURE_ARGS
@@ -289,28 +277,12 @@ hadrian-ghc-in-ghci:
   <<: *only-default
   variables:
     TEST_TYPE: test
-  before_script:
-    - git clean -xdf && git submodule foreach git clean -xdf
+    MAKE_ARGS: "-Werror"
   script:
-    - ./boot
-    - ./configure $CONFIGURE_ARGS
-    - |
-      THREADS=`mk/detect-cpu-count.sh`
-      make V=0 -j$THREADS WERROR=-Werror
-    - make binary-dist-prep TAR_COMP_OPTS="-1"
-    - make test_bindist TEST_PREP=YES
-    - |
-      # Prepare to push git notes.
-      METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
-      git config user.email "ben+ghc-ci@smart-cactus.org"
-      git config user.name "GHC GitLab CI"
-      .gitlab/test-metrics.sh pull
-    - |
-      THREADS=`mk/detect-cpu-count.sh`
-      make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE || (METRICS_FILE=$METRICS_FILE .gitlab/test-metrics.sh push && false)
-    - |
-      # Push git notes.
-      METRICS_FILE=$METRICS_FILE .gitlab/test-metrics.sh push
+    - .gitlab/ci.sh setup
+    - .gitlab/ci.sh configure
+    - .gitlab/ci.sh build_make
+    - .gitlab/ci.sh test_make
   dependencies: []
   artifacts:
     reports:
@@ -322,6 +294,79 @@ hadrian-ghc-in-ghci:
       - performance-metrics.tsv
 
 #################################
+# x86_64-freebsd
+#################################
+
+.build-x86_64-freebsd:
+  extends: .validate
+  tags:
+    - x86_64-freebsd
+  allow_failure: true
+  variables:
+    # N.B. we use iconv from ports as I see linker errors when we attempt
+    # to use the "native" iconv embedded in libc as suggested by the
+    # porting guide [1].
+    # [1] https://www.freebsd.org/doc/en/books/porters-handbook/using-iconv.html)
+    CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib"
+    GHC_VERSION: 8.6.3
+    CABAL_INSTALL_VERSION: 3.0.0.0
+    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz"
+    TEST_ENV: "x86_64-freebsd"
+    BUILD_FLAVOUR: "validate"
+  after_script:
+    - cp -Rf $HOME/.cabal cabal-cache
+    - .gitlab/ci.sh clean
+  artifacts:
+    when: always
+    expire_in: 2 week
+  cache:
+    key: "freebsd-$GHC_VERSION"
+    paths:
+      - cabal-cache
+      - toolchain
+
+# Disabled due to lack of builder capacity
+.validate-x86_64-freebsd:
+  extends: .build-x86_64-freebsd
+  stage: full-build
+
+nightly-x86_64-freebsd:
+  <<: *nightly
+  extends: .build-x86_64-freebsd
+  stage: full-build
+
+.build-x86_64-freebsd-hadrian:
+  extends: .validate-hadrian
+  stage: full-build
+  tags:
+    - x86_64-freebsd
+  allow_failure: true
+  variables:
+    CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib"
+    HADRIAN_ARGS: "--docs=no-sphinx"
+    GHC_VERSION: 8.6.3
+    CABAL_INSTALL_VERSION: 3.0.0.0
+    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz"
+    TEST_ENV: "x86_64-freebsd-hadrian"
+    FLAVOUR: "validate"
+  after_script:
+    - cp -Rf $HOME/.cabal cabal-cache
+    - .gitlab/ci.sh clean
+  artifacts:
+    when: always
+    expire_in: 2 week
+  cache:
+    key: "freebsd-$GHC_VERSION"
+    paths:
+      - cabal-cache
+      - toolchain
+
+# Disabled due to lack of builder capacity
+.validate-x86_64-freebsd-hadrian:
+  extends: .build-x86_64-freebsd-hadrian
+  stage: full-build
+
+#################################
 # x86_64-darwin
 #################################
 
@@ -332,25 +377,18 @@ validate-x86_64-darwin:
     - x86_64-darwin
   variables:
     GHC_VERSION: 8.6.5
-    CABAL_INSTALL_VERSION: 2.4.1.0
+    CABAL_INSTALL_VERSION: 3.0.0.0
     BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-apple-darwin.tar.xz"
     MACOSX_DEPLOYMENT_TARGET: "10.7"
     # Only Sierra and onwards supports clock_gettime. See #12858
     ac_cv_func_clock_gettime: "no"
     LANG: "en_US.UTF-8"
-    CONFIGURE_ARGS: --with-intree-gmp
+    CONFIGURE_ARGS: "--with-intree-gmp"
     TEST_ENV: "x86_64-darwin"
-  before_script:
-    - git clean -xdf && git submodule foreach git clean -xdf
-    - git submodule sync --recursive
-    - git submodule update --init --recursive
-    - git checkout .gitmodules
-    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
-
-    - bash .gitlab/darwin-init.sh
-    - PATH="`pwd`/toolchain/bin:$PATH"
+    BUILD_FLAVOUR: "perf"
   after_script:
     - cp -Rf $HOME/.cabal cabal-cache
+    - .gitlab/ci.sh clean
   artifacts:
     when: always
     expire_in: 2 week
@@ -374,26 +412,14 @@ validate-x86_64-darwin:
     CONFIGURE_ARGS: --with-intree-gmp
     TEST_ENV: "x86_64-darwin-hadrian"
     FLAVOUR: "validate"
-  before_script:
-    - git clean -xdf && git submodule foreach git clean -xdf
-    - git submodule sync --recursive
-    - git submodule update --init --recursive
-    - git checkout .gitmodules
-    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
-
-    - bash .gitlab/darwin-init.sh
-    - PATH="`pwd`/toolchain/bin:$PATH"
   script:
-    - cabal update
-    - ./boot
-    - ./configure $CONFIGURE_ARGS
-    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
-    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
-    - export TOP=$(pwd)
-    - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../
-    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc
+    - .gitlab/ci.sh setup
+    - .gitlab/ci.sh configure
+    - .gitlab/ci.sh build_hadrian
+    - .gitlab/ci.sh test_hadrian
   after_script:
     - cp -Rf $HOME/.cabal cabal-cache
+    - .gitlab/ci.sh clean
   artifacts:
     when: always
     expire_in: 2 week
@@ -407,19 +433,15 @@ validate-x86_64-darwin:
   extends: .validate
   tags:
     - x86_64-linux
+  variables:
+    BUILD_FLAVOUR: "perf"
   before_script:
-    - git clean -xdf && git submodule foreach git clean -xdf
-    - git submodule sync --recursive
-    - git submodule update --init --recursive
-    - git checkout .gitmodules
-    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
       # Build hyperlinked sources for documentation when building releases
     - |
       if [[ -n "$CI_COMMIT_TAG" ]]; then
-        echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
+        HADDOCK_HYPERLINKED_SOURCES=1
       fi
 
-    - .gitlab/prepare-system.sh
     # workaround for docker permissions
     - sudo chown ghc:ghc -R .
   after_script:
@@ -771,57 +793,44 @@ validate-x86_64-linux-fedora27:
   before_script:
     - git clean -xdf
 
-    # Use a local temporary directory to ensure that concurrent builds don't
-    # interfere with one another
-    - |
-      mkdir tmp
-      set TMP=%cd%\tmp
-      set TEMP=%cd%\tmp
-
-    - set PATH=C:\msys64\usr\bin;%PATH%
-    # On Windows submodules can inexplicably get into funky states where git
-    # believes that the submodule is initialized yet its associated repository
-    # is not valid. Avoid failing in this case with the following insanity.
-    - |
-      bash -c 'git submodule sync --recursive || git submodule deinit --force --all'
-      bash -c 'git submodule update --init --recursive'
-      git submodule foreach git clean -xdf
-    - git checkout .gitmodules
-    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
-    - bash .gitlab/win32-init.sh
+    # Setup toolchain
+    - bash .gitlab/ci.sh setup
   after_script:
-    - rd /s /q tmp
-    - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
-    - bash -c 'make clean || true'
+    - |
+      Copy-Item -Recurse -Path $Env:APPDATA\cabal -Destination cabal-cache
+    - bash .gitlab/ci.sh clean
   dependencies: []
   variables:
     #FORCE_SYMLINKS: 1
     LANG: "en_US.UTF-8"
     SPHINXBUILD: "/mingw64/bin/sphinx-build.exe"
+    CABAL_INSTALL_VERSION: 3.0.0.0
+    GHC_VERSION: "8.6.5"
   cache:
     paths:
       - cabal-cache
-      - ghc-8.6.5
+      - toolchain
       - ghc-tarballs
 
 .build-windows-hadrian:
   extends: .build-windows
   stage: full-build
   variables:
-    GHC_VERSION: "8.6.5"
     FLAVOUR: "validate"
+    # skipping perf tests for now since we build a quick-flavoured GHC,
+    # which might result in some broken perf tests?
+    HADRIAN_ARGS: "--docs=no-sphinx --skip-perf"
+
   # due to #16574 this currently fails
   allow_failure: true
+
   script:
-    - |
-      python boot
-      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
-    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist"
-    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
-    - bash -c "export TOP=$(pwd); cd _build/bindist/ghc-*/ && PATH=$TOP/toolchain/bin:$PATH ./configure --prefix=$TOP/_build/install && make install && cd ../../../"
-    - bash -c "export TOP=$(pwd); PATH=$TOP/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc"
+    - bash .gitlab/ci.sh configure
+    - bash .gitlab/ci.sh build_hadrian
+    - bash .gitlab/ci.sh test_hadrian
   tags:
-    - x86_64-windows
+    - new-x86_64-windows
+    - test
   artifacts:
     reports:
       junit: junit.xml
@@ -833,7 +842,6 @@ validate-x86_64-linux-fedora27:
 
 validate-x86_64-windows-hadrian:
   extends: .build-windows-hadrian
-  stage: full-build
   variables:
     MSYSTEM: MINGW64
     TEST_ENV: "x86_64-windows-hadrian"
@@ -856,20 +864,14 @@ nightly-i386-windows-hadrian:
   stage: full-build
   variables:
     BUILD_FLAVOUR: "quick"
-    GHC_VERSION: "8.6.5"
     BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-mingw32.tar.xz"
   script:
-    - |
-      python boot
-      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex $CONFIGURE_ARGS'
-    - bash -c "echo include mk/flavours/${BUILD_FLAVOUR}.mk > mk/build.mk"
-    - bash -c "echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk"
-    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
-    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist-prep TAR_COMP_OPTS=-1"
-    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make test_bindist TEST_PREP=YES"
-    - bash -c 'make V=0 test PYTHON=/mingw64/bin/python3 THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
+    - bash .gitlab/ci.sh configure
+    - bash .gitlab/ci.sh build_make
+    - bash .gitlab/ci.sh test_make
   tags:
-    - x86_64-windows
+    - new-x86_64-windows
+    - test
   artifacts:
     when: always
     expire_in: 2 week
@@ -881,64 +883,60 @@ nightly-i386-windows-hadrian:
       - "ghc-x86_64-mingw32.tar.xz"
       - junit.xml
 
-validate-x86_64-windows:
+.build-x86_64-windows-make:
   extends: .build-windows-make
   variables:
     MSYSTEM: MINGW64
-    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
     TEST_ENV: "x86_64-windows"
   cache:
     key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"
 
+validate-x86_64-windows:
+  extends: .build-x86_64-windows-make
+
 nightly-x86_64-windows:
-  extends: .build-windows-make
+  extends: .build-x86_64-windows-make
   stage: full-build
   variables:
     BUILD_FLAVOUR: "validate"
-    MSYSTEM: MINGW64
-    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
   only:
     variables:
       - $NIGHTLY
-  cache:
-    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"
 
 # Normal Windows validate builds are profiled; that won't do for releases.
 release-x86_64-windows:
   <<: *release
   extends: validate-x86_64-windows
   variables:
-    MSYSTEM: MINGW64
     BUILD_FLAVOUR: "perf"
-    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
-    TEST_ENV: "x86_64-windows"
 
-release-i386-windows:
-  <<: *release
+
+.build-i386-windows-make:
   extends: .build-windows-make
   variables:
     MSYSTEM: MINGW32
-    BUILD_FLAVOUR: "perf"
-    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
     # Due to #15934
     BUILD_PROF_LIBS: "NO"
     TEST_ENV: "i386-windows"
   cache:
     key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
 
+validate-i386-windows:
+  extends: .build-i386-windows-make
+  variables:
+    BUILD_FLAVOUR: "perf"
+
+release-i386-windows:
+  <<: *release
+  extends: .build-i386-windows-make
+  variables:
+    BUILD_FLAVOUR: "perf"
+
 nightly-i386-windows:
-  extends: .build-windows-make
+  extends: .build-i386-windows-make
   only:
     variables:
       - $NIGHTLY
-  variables:
-    MSYSTEM: MINGW32
-    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
-    # Due to #15934
-    BUILD_PROF_LIBS: "NO"
-    TEST_ENV: "i386-windows"
-  cache:
-    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
 
 ############################################################
 # Cleanup
diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh
new file mode 100755
index 0000000000..90d789f040
--- /dev/null
+++ b/.gitlab/ci.sh
@@ -0,0 +1,428 @@
+#!/usr/bin/env bash
+
+# This is the primary driver of the GitLab CI infrastructure.
+
+set -e -o pipefail
+
+# Configuration:
+hackage_index_state="@1579718451"
+
+# Colors
+BLACK="0;30"
+GRAY="1;30"
+RED="0;31"
+LT_RED="1;31"
+BROWN="0;33"
+LT_BROWN="1;33"
+GREEN="0;32"
+LT_GREEN="1;32"
+BLUE="0;34"
+LT_BLUE="1;34"
+PURPLE="0;35"
+LT_PURPLE="1;35"
+CYAN="0;36"
+LT_CYAN="1;36"
+WHITE="1;37"
+LT_GRAY="0;37"
+
+# GitLab Pipelines log section delimiters
+# https://gitlab.com/gitlab-org/gitlab-foss/issues/14664
+start_section() {
+  name="$1"
+  echo -e "section_start:$(date +%s):$name\015\033[0K"
+}
+
+end_section() {
+  name="$1"
+  echo -e "section_end:$(date +%s):$name\015\033[0K"
+}
+
+echo_color() {
+  local color="$1"
+  local msg="$2"
+  echo -e "\e[${color}m${msg}\e[0m"
+}
+
+error() { echo_color "${RED}" "$1"; }
+warn() { echo_color "${LT_BROWN}" "$1"; }
+info() { echo_color "${LT_BLUE}" "$1"; }
+
+fail() { error "error: $1"; exit 1; }
+
+function run() {
+  info "Running $*..."
+  "$@" || ( error "$* failed"; return 1; )
+}
+
+if [ -z "$GHC_VERSION" ]; then
+  fail "GHC_VERSION not set"
+fi
+
+TOP="$(pwd)"
+
+function mingw_init() {
+  case "$MSYSTEM" in
+    MINGW32)
+      triple="i386-unknown-mingw32"
+      boot_triple="i386-unknown-mingw32" # triple of bootstrap GHC
+      ;;
+    MINGW64)
+      triple="x86_64-unknown-mingw32"
+      boot_triple="x86_64-unknown-mingw32" # triple of bootstrap GHC
+      ;;
+    *)
+      fail "win32-init: Unknown MSYSTEM $MSYSTEM"
+      ;;
+  esac
+
+  # Bring mingw toolchain into PATH.
+  # This is extracted from /etc/profile since this script inexplicably fails to
+  # run under gitlab-runner.
+  source /etc/msystem
+  MINGW_MOUNT_POINT="${MINGW_PREFIX}"
+  PATH="$MINGW_MOUNT_POINT/bin:$PATH"
+}
+
+# This will contain GHC's local native toolchain
+toolchain="$TOP/toolchain"
+mkdir -p $toolchain/bin
+PATH="$toolchain/bin:$PATH"
+
+export METRICS_FILE="$CI_PROJECT_DIR/performance-metrics.tsv"
+
+# Use a local temporary directory to ensure that concurrent builds don't
+# interfere with one another
+mkdir -p $TOP/tmp
+export TMP="$TOP/tmp"
+export TEMP="$TOP/tmp"
+
+function darwin_setup() {
+  # It looks like we already have python2 here and just installing python3
+  # does not work.
+  brew upgrade python
+  brew install ghc cabal-install ncurses gmp
+
+  pip3 install sphinx
+  # PDF documentation disabled as MacTeX apparently doesn't include xelatex.
+  #brew cask install mactex
+}
+
+function show_tool() {
+  local tool="$1"
+  info "$tool = ${!tool}"
+  ${!tool} --version
+}
+
+function set_toolchain_paths() {
+  needs_toolchain=1
+  case "$(uname)" in
+    Linux) needs_toolchain="" ;;
+    *) ;;
+  esac
+
+  if [[ -n "$needs_toolchain" ]]; then
+      # These are populated by setup_toolchain
+      export GHC="$toolchain/bin/ghc$exe"
+      export CABAL="$toolchain/bin/cabal$exe"
+      export HAPPY="$toolchain/bin/happy$exe"
+      export ALEX="$toolchain/bin/alex$exe"
+  else
+      export GHC="$(which ghc)"
+      export CABAL="/usr/local/bin/cabal"
+      export HAPPY="$HOME/.cabal/bin/happy"
+      export ALEX="$HOME/.cabal/bin/alex"
+  fi
+
+  # FIXME: Temporarily use ghc from ports
+  case "$(uname)" in
+    FreeBSD) GHC="/usr/local/bin/ghc" ;;
+    *) ;;
+  esac
+}
+
+# Extract GHC toolchain
+function setup() {
+  if [ -d "$TOP/cabal-cache" ]; then
+      info "Extracting cabal cache..."
+      cp -Rf cabal-cache $cabal_dir
+  fi
+
+  if [[ -n "$needs_toolchain" ]]; then
+    setup_toolchain
+  fi
+  case "$(uname)" in
+    Darwin) darwin_setup ;;
+    *) ;;
+  esac
+
+  # Make sure that git works
+  git config user.email "ghc-ci@gitlab-haskell.org"
+  git config user.name "GHC GitLab CI"
+
+  info "====================================================="
+  info "Toolchain versions"
+  info "====================================================="
+  show_tool GHC
+  show_tool CABAL
+  show_tool HAPPY
+  show_tool ALEX
+}
+
+function fetch_ghc() {
+  local v="$GHC_VERSION"
+  if [[ -z "$v" ]]; then
+      fail "GHC_VERSION is not set"
+  fi
+
+  if [ ! -e "$GHC" ]; then
+      start_section "fetch GHC"
+      url="https://downloads.haskell.org/~ghc/${GHC_VERSION}/ghc-${GHC_VERSION}-${boot_triple}.tar.xz"
+      info "Fetching GHC binary distribution from $url..."
+      curl $url > ghc.tar.xz || fail "failed to fetch GHC binary distribution"
+      tar -xJf ghc.tar.xz || fail "failed to extract GHC binary distribution"
+      case "$(uname)" in
+        MSYS_*|MINGW*)
+          cp -r ghc-${GHC_VERSION}/* $toolchain
+          ;;
+        *)
+          pushd ghc-${GHC_VERSION}
+          ./configure --prefix=$toolchain
+          $MAKE install
+          popd
+          ;;
+      esac
+      rm -Rf ghc-${GHC_VERSION} ghc.tar.xz
+      end_section "fetch GHC"
+  fi
+
+}
+
+function fetch_cabal() {
+  local v="$CABAL_INSTALL_VERSION"
+  if [[ -z "$v" ]]; then
+      fail "CABAL_INSTALL_VERSION is not set"
+  fi
+
+  if [ ! -e "$CABAL" ]; then
+      start_section "fetch GHC"
+      case "$(uname)" in
+        # N.B. Windows uses zip whereas all others use .tar.xz
+        MSYS_*|MINGW*)
+          case "$MSYSTEM" in
+            MINGW32) cabal_arch="i386" ;;
+            MINGW64) cabal_arch="x86_64" ;;
+            *) fail "unknown MSYSTEM $MSYSTEM" ;;
+          esac
+          url="https://downloads.haskell.org/~cabal/cabal-install-$v/cabal-install-$v-$cabal_arch-unknown-mingw32.zip"
+          info "Fetching cabal binary distribution from $url..."
+          curl $url > $TMP/cabal.zip
+          unzip $TMP/cabal.zip
+          mv cabal.exe $CABAL
+          ;;
+        *)
+          local base_url="https://downloads.haskell.org/~cabal/cabal-install-$v/"
+          case "$(uname)" in
+            Darwin) cabal_url="$base_url/cabal-install-$v-x86_64-apple-darwin17.7.0.tar.xz" ;;
+            FreeBSD)
+              #cabal_url="$base_url/cabal-install-$v-x86_64-portbld-freebsd.tar.xz" ;;
+              cabal_url="http://home.smart-cactus.org/~ben/ghc/cabal-install-3.0.0.0-x86_64-portbld-freebsd.tar.xz" ;;
+            *) fail "don't know where to fetch cabal-install for $(uname)"
+          esac
+          echo "Fetching cabal-install from $cabal_url"
+          curl $cabal_url > cabal.tar.xz
+          tar -xJf cabal.tar.xz
+          mv cabal $toolchain/bin
+          ;;
+      esac
+      end_section "fetch GHC"
+  fi
+}
+
+# For non-Docker platforms we prepare the bootstrap toolchain
+# here. For Docker platforms this is done in the Docker image
+# build.
+function setup_toolchain() {
+  fetch_ghc
+  fetch_cabal
+  cabal_install="$CABAL v2-install --index-state=$hackage_index_state --installdir=$toolchain/bin"
+  # Avoid symlinks on Windows
+  case "$(uname)" in
+    MSYS_*|MINGW*) cabal_install="$cabal_install --install-method=copy" ;;
+    *) ;;
+  esac
+
+  if [ ! -e "$HAPPY" ]; then
+      info "Building happy..."
+      cabal update
+      $cabal_install happy
+  fi
+
+  if [ ! -e "$ALEX" ]; then
+      info "Building alex..."
+      cabal update
+      $cabal_install alex
+  fi
+}
+
+function cleanup_submodules() {
+  start_section "clean submodules"
+  info "Cleaning submodules..."
+  # On Windows submodules can inexplicably get into funky states where git
+  # believes that the submodule is initialized yet its associated repository
+  # is not valid. Avoid failing in this case with the following insanity.
+  git submodule sync --recursive || git submodule deinit --force --all
+  git submodule update --init --recursive
+  git submodule foreach git clean -xdf
+  end_section "clean submodules"
+}
+
+function prepare_build_mk() {
+  if [[ -z "$BUILD_FLAVOUR" ]]; then fail "BUILD_FLAVOUR is not set"; fi
+  if [[ -z ${BUILD_SPHINX_HTML:-} ]]; then BUILD_SPHINX_HTML=YES; fi
+  if [[ -z ${BUILD_SPHINX_PDF:-} ]]; then BUILD_SPHINX_PDF=YES; fi
+  if [[ -z ${INTEGER_LIBRARY:-} ]]; then INTEGER_LIBRARY=integer-gmp; fi
+
+  cat > mk/build.mk <<EOF
+V=1
+HADDOCK_DOCS=YES
+LATEX_DOCS=YES
+HSCOLOUR_SRCS=YES
+BUILD_SPHINX_HTML=$BUILD_SPHINX_HTML
+BUILD_SPHINX_PDF=$BUILD_SPHINX_PDF
+BeConservative=YES
+INTEGER_LIBRARY=$INTEGER_LIBRARY
+XZ_CMD=$XZ
+
+BuildFlavour=$BUILD_FLAVOUR
+ifneq "\$(BuildFlavour)" ""
+include mk/flavours/\$(BuildFlavour).mk
+endif
+GhcLibHcOpts+=-haddock
+EOF
+
+  if [ -n "$HADDOCK_HYPERLINKED_SOURCES" ]; then
+    echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
+  fi
+
+  case "$(uname)" in
+    Darwin) echo "libraries/integer-gmp_CONFIGURE_OPTS += --configure-option=--with-intree-gmp" >> mk/build.mk ;;
+    *) ;;
+  esac
+
+  info "build.mk is:"
+  cat mk/build.mk
+}
+
+function configure() {
+  start_section "booting"
+  run python3 boot
+  end_section "booting"
+
+  start_section "configuring"
+  run ./configure \
+    --enable-tarballs-autodownload \
+    --target=$triple \
+    $CONFIGURE_ARGS \
+    GHC=$GHC \
+    HAPPY=$HAPPY \
+    ALEX=$ALEX \
+    || ( cat config.log; fail "configure failed" )
+  end_section "configuring"
+}
+
+function build_make() {
+  prepare_build_mk
+  if [[ -z "$BIN_DIST_PREP_TAR_COMP" ]]; then
+    fail "BIN_DIST_PREP_TAR_COMP is not set"
+  fi
+
+  echo "include mk/flavours/${BUILD_FLAVOUR}.mk" > mk/build.mk
+  echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk
+  run $MAKE -j$(mk/detect-cpu-count.sh) $MAKE_ARGS
+  run $MAKE -j$(mk/detect-cpu-count.sh) binary-dist-prep TAR_COMP_OPTS=-1
+  ls -lh $BIN_DIST_PREP_TAR_COMP
+}
+
+function fetch_perf_notes() {
+  info "Fetching perf notes..."
+  $TOP/.gitlab/test-metrics.sh pull
+}
+
+function push_perf_notes() {
+  info "Pushing perf notes..."
+  $TOP/.gitlab/test-metrics.sh push
+}
+
+function test_make() {
+  run $MAKE test_bindist TEST_PREP=YES
+  run $MAKE V=0 test \
+    THREADS=$(mk/detect-cpu-count.sh) \
+    JUNIT_FILE=../../junit.xml
+}
+
+function build_hadrian() {
+  if [ -z "$FLAVOUR" ]; then
+    fail "FLAVOUR not set"
+  fi
+
+  run hadrian/build.cabal.sh \
+    --flavour=$FLAVOUR \
+    -j$(mk/detect-cpu-count.sh) \
+    $HADRIAN_ARGS \
+    binary-dist
+
+  mv _build/bindist/ghc*.tar.xz ghc.tar.xz
+}
+
+function test_hadrian() {
+  cd _build/bindist/ghc-*/
+  run ./configure --prefix=$TOP/_build/install
+  run $MAKE install
+  cd ../../../
+
+  run hadrian/build.cabal.sh \
+    --flavour=$FLAVOUR \
+    -j$(mk/detect-cpu-count.sh) \
+    $HADRIAN_ARGS \
+    test \
+    --summary-junit=./junit.xml \
+    --test-compiler=$TOP/_build/install/bin/ghc
+}
+
+function clean() {
+  rm -R tmp
+  run $MAKE --quiet clean || true
+  run rm -Rf _build
+}
+
+# Determine Cabal data directory
+case "$(uname)" in
+  MSYS_*|MINGW*) exe=".exe"; cabal_dir="$APPDATA/cabal" ;;
+  *) cabal_dir="$HOME/.cabal"; exe="" ;;
+esac
+
+# Platform-specific environment initialization
+MAKE="make"
+case "$(uname)" in
+  MSYS_*|MINGW*) mingw_init ;;
+  Darwin) boot_triple="x86_64-apple-darwin" ;;
+  FreeBSD)
+    boot_triple="x86_64-portbld-freebsd"
+    MAKE="gmake"
+    ;;
+  Linux) ;;
+  *) fail "uname $(uname) is not supported" ;;
+esac
+
+set_toolchain_paths
+
+case $1 in
+  setup) setup && cleanup_submodules ;;
+  configure) configure ;;
+  build_make) build_make ;;
+  test_make) fetch_perf_notes; test_make; push_perf_notes ;;
+  build_hadrian) build_hadrian ;;
+  test_hadrian) fetch_perf_notes; test_hadrian; push_perf_notes ;;
+  clean) clean ;;
+  *) fail "unknown mode $1" ;;
+esac
diff --git a/.gitlab/darwin-init.sh b/.gitlab/darwin-init.sh
deleted file mode 100644
index 3dadc04dc1..0000000000
--- a/.gitlab/darwin-init.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-set -e
-
-toolchain=`pwd`/toolchain
-PATH="$toolchain/bin:$PATH"
-
-if [ -d "`pwd`/cabal-cache" ]; then
-    cp -Rf cabal-cache $HOME/.cabal
-fi
-
-if [ ! -e $toolchain/bin/ghc ]; then
-    mkdir -p tmp
-    cd tmp
-    ghc_tarball="https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-x86_64-apple-darwin.tar.xz"
-    echo "Fetching GHC from $ghc_tarball"
-    curl $ghc_tarball | tar -xJ
-    cd ghc-$GHC_VERSION
-    ./configure --prefix=$toolchain
-    make install
-    cd ../..
-    rm -Rf tmp
-fi
-
-if [ ! -e $toolchain/bin/cabal ]; then
-    cabal_tarball="https://downloads.haskell.org/~cabal/cabal-install-$CABAL_INSTALL_VERSION/cabal-install-$CABAL_INSTALL_VERSION-x86_64-apple-darwin-sierra.tar.xz"
-    echo "Fetching cabal-install from $cabal_tarball"
-    curl $cabal_tarball | tar -xz
-    mv cabal $toolchain/bin
-fi
-
-if [ ! -e $toolchain/bin/happy ]; then
-    cabal update
-    cabal new-install happy --symlink-bindir=$toolchain/bin
-fi
-
-if [ ! -e $toolchain/bin/alex ]; then
-    cabal update
-    cabal new-install alex --symlink-bindir=$toolchain/bin
-fi
-
diff --git a/.gitlab/prepare-system.sh b/.gitlab/prepare-system.sh
deleted file mode 100755
index 4180e6029c..0000000000
--- a/.gitlab/prepare-system.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-# vim: sw=2 et
-set -euo pipefail
-
-fail() {
-  echo "ERROR: $*" >&2
-  exit 1
-}
-
-hackage_index_state="@1522046735"
-
-if [[ -z ${BUILD_SPHINX_HTML:-} ]]; then BUILD_SPHINX_HTML=YES; fi
-if [[ -z ${BUILD_SPHINX_PDF:-} ]]; then BUILD_SPHINX_PDF=YES; fi
-if [[ -z ${INTEGER_LIBRARY:-} ]]; then INTEGER_LIBRARY=integer-gmp; fi
-if [[ -z ${BUILD_FLAVOUR:-} ]]; then BUILD_FLAVOUR=perf; fi
-
-if [[ -z ${XZ:-} ]]; then
-  if which pxz; then
-    XZ="pxz"
-  elif which xz; then
-    # Check whether --threads is supported
-    if echo "hello" | xz --threads=$CORES >/dev/null; then
-      XZ="xz --threads=$CORES"
-    else
-      XZ="xz"
-    fi
-  else
-    echo "error: neither pxz nor xz were found"
-    exit 1
-  fi
-fi
-echo "Using $XZ for compression..."
-
-
-cat > mk/build.mk <<EOF
-V=1
-HADDOCK_DOCS=YES
-LATEX_DOCS=YES
-HSCOLOUR_SRCS=YES
-BUILD_SPHINX_HTML=$BUILD_SPHINX_HTML
-BUILD_SPHINX_PDF=$BUILD_SPHINX_PDF
-BeConservative=YES
-INTEGER_LIBRARY=$INTEGER_LIBRARY
-XZ_CMD=$XZ
-EOF
-
-cat <<EOF >> mk/build.mk
-BuildFlavour=$BUILD_FLAVOUR
-ifneq "\$(BuildFlavour)" ""
-include mk/flavours/\$(BuildFlavour).mk
-endif
-GhcLibHcOpts+=-haddock
-EOF
-
-case "$(uname)" in
-  Linux)
-    if [[ -n ${TARGET:-} ]]; then
-      if [[ $TARGET = FreeBSD ]]; then
-        # cross-compiling to FreeBSD
-        echo 'HADDOCK_DOCS = NO' >> mk/build.mk
-        echo 'WERROR=' >> mk/build.mk
-        # https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables
-        echo 'export PATH=/opt/ghc/bin:$PATH' >> $BASH_ENV
-      else
-        fail "TARGET=$target not supported"
-      fi
-    fi
-    ;;
-
-  Darwin)
-    if [[ -n ${TARGET:-} ]]; then
-      fail "uname=$(uname) not supported for cross-compilation"
-    fi
-    # It looks like we already have python2 here and just installing python3
-    # does not work.
-    brew upgrade python
-    brew install ghc cabal-install ncurses gmp
-
-    pip3 install sphinx
-    # PDF documentation disabled as MacTeX apparently doesn't include xelatex.
-    #brew cask install mactex
-
-    cabal update
-    cabal install --reinstall alex happy haddock hscolour --index-state=$hackage_index_state
-    # put them on the $PATH, don't fail if already installed
-    ln -s $HOME/.cabal/bin/alex /usr/local/bin/alex || true
-    ln -s $HOME/.cabal/bin/happy /usr/local/bin/happy || true
-    ln -s $HOME/.cabal/bin/HsColour /usr/local/bin/HsColour || true
-    echo "libraries/integer-gmp_CONFIGURE_OPTS += --configure-option=--with-intree-gmp" >> mk/build.mk
-    ;;
-  *)
-    fail "uname=$(uname) not supported"
-esac
-
-echo "================================================="
-echo "Build.mk:"
-echo ""
-cat mk/build.mk
-echo "================================================="
diff --git a/.gitlab/win32-init.sh b/.gitlab/win32-init.sh
deleted file mode 100644
index aec73ce083..0000000000
--- a/.gitlab/win32-init.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-
-set -e
-
-toolchain=`pwd`/toolchain
-PATH="$toolchain/bin:/mingw64/bin:$PATH"
-
-if [ -d "`pwd`/cabal-cache" ]; then
-    cp -Rf cabal-cache $APPDATA/cabal
-fi
-
-if [ ! -e $toolchain/bin/ghc ]; then
-    case $MSYSTEM in
-      MINGW32)
-        triple="i386-unknown-mingw32"
-        ;;
-      MINGW64)
-        triple="x86_64-unknown-mingw32"
-        ;;
-      *)
-        echo "win32-init: Unknown MSYSTEM $MSYSTEM"
-        exit 1
-        ;;
-    esac
-    curl https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-$triple.tar.xz | tar -xJ
-    mv ghc-$GHC_VERSION toolchain
-fi
-
-if [ ! -e $toolchain/bin/cabal ]; then
-    url="https://downloads.haskell.org/~cabal/cabal-install-2.4.1.0/cabal-install-2.4.1.0-x86_64-unknown-mingw32.zip"
-    curl $url > /tmp/cabal.zip
-    unzip /tmp/cabal.zip
-    mv cabal.exe $toolchain/bin
-fi
-
-if [ ! -e $toolchain/bin/happy ]; then
-    cabal update
-    cabal install happy
-    cp $APPDATA/cabal/bin/happy $toolchain/bin
-fi
-
-if [ ! -e $toolchain/bin/alex ]; then
-    cabal update
-    cabal install alex
-    cp $APPDATA/cabal/bin/alex $toolchain/bin
-fi
-