[ci/arm/darwin/testsuite] Forwards ports from GHC-8.10

This is a set of forward ports (cherry-picks) from 8.10

- a7d22795ed [ci] Add support for building on aarch64-darwin
- 5109e87e13 [testlib/driver] denoise
- 307d34945b [ci] default value for CONFIGURE_ARGS
- 10a18cb4e0 [testsuite] mark ghci056 as fragile
- 16c13d5acf [ci] Default value for MAKE_ARGS
- ab571457b9 [ci/build] Copy config.sub around
- 251892b98f [ci/darwin] bump nixpkgs rev
- 5a6c36ecb4 [testsuite/darwin] fix conc059
- aae95ef0c9 [ci] add timing info
- 3592d1104c [Aarch64] No div-by-zero; disable test.
- 57671071ad [Darwin] mark stdc++ tests as broken
- 33c4d49754 [testsuite] filter out superfluous dylib warnings
- 4bea83afec [ci/nix-shell] Add Foundation and Security
- 6345530062 [testsuite/json2] Fix failure with LLVM backends
- c3944bc89d [ci/nix-shell] [Darwin] Stop the ld warnings about libiconv.
- b821fcc714 [testsuite] static001 is not broken anymore.
- f7062e1b0c [testsuite/arm64] fix section_alignment
- 820b076698 [darwin] stop the DYLD_LIBRARY_PATH madness
- 07b1af0362 [ci/nix-shell] uniquify NIX_LDFLAGS{_FOR_TARGET}

As well as a few additional fixups needed to make this block compile:
- Fixup all.T
- Set CROSS_TARGET, BROKEN_TESTS, XZ, RUNTEST_ARGS, default value.
- [ci] shell.nix bump happy

14 files changed, 340 insertions, 64 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ca0eb3b842..72cb116556 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -383,11 +383,42 @@ lint-base:
   variables:
     TEST_TYPE: test
     MAKE_ARGS: "-Werror"
-  script:
-    - .gitlab/ci.sh setup
-    - .gitlab/ci.sh configure
-    - .gitlab/ci.sh build_make
-    - .gitlab/ci.sh test_make
+  script: |
+    # Build hyperlinked sources for documentation when building releases
+    if [[ "$RELEASE_JOB" = "yes" ]]; then
+      HADDOCK_HYPERLINKED_SOURCES=1
+    fi
+    TIME_START=$(date +%s)
+    time .gitlab/ci.sh setup 2>&1
+    TIME_SETUP=$(date +%s)
+    TIME_SETUP_DELTA=$(expr $TIME_SETUP - $TIME_START)
+    echo "Setup took $TIME_SETUP_DELTA seconds"
+
+    time .gitlab/ci.sh configure 2>&1
+    TIME_CONFIGURE=$(date +%s)
+    TIME_CONFIGURE_DELTA=$(expr $TIME_CONFIGURE - $TIME_SETUP)
+    echo "Configure took $TIME_CONFIGURE_DELTA seconds"
+
+    time .gitlab/ci.sh build_make 2>&1
+    TIME_BUILD=$(date +%s)
+    TIME_BUILD_DELTA=$(expr $TIME_BUILD - $TIME_CONFIGURE)
+    echo "Build took $TIME_BUILD_DELTA seconds"
+
+    set +e
+    time .gitlab/ci.sh test_make 2>&1
+    status=$?
+    set -e
+    TIME_TEST=$(date +%s)
+    TIME_TEST_DELTA=$(expr $TIME_TEST - $TIME_BUILD)
+    echo "Test took $TIME_TEST_DELTA seconds"
+
+    echo "=== TIMINGS ==="
+    echo "Setup      | $TIME_SETUP_DELTA"
+    echo "Configure  | $TIME_CONFIGURE_DELTA"
+    echo "Build      | $TIME_BUILD_DELTA"
+    echo "Test       | $TIME_TEST_DELTA"
+    exit $status
+
   dependencies: []
   artifacts:
     reports:
@@ -481,10 +512,13 @@ release-x86_64-freebsd:
 
 validate-x86_64-darwin:
   extends: .validate
-  needs: [validate-x86_64-linux-deb9-hadrian]
   stage: full-build
   tags:
-    - x86_64-darwin
+    - x86_64-darwin-m1
+
+  # for now make this non mandatory to pass.
+  allow_failure: true
+
   variables:
     GHC_VERSION: 8.10.4
     CABAL_INSTALL_VERSION: 3.2.0.0
@@ -494,7 +528,15 @@ validate-x86_64-darwin:
     ac_cv_func_clock_gettime: "no"
     # Only newer OS Xs support utimensat. See #17895
     ac_cv_func_utimensat: "no"
+
+    TEST_TYPE: test
+    MAKE_ARGS: "-Werror"
+    # we run on M1's for now, getconf can't be built with nix yet,
+    # and we use a pure shell, so we can't/shouldn't use /usr/bin/getconf
+    # inside th shell.
+    CPUS: 8
     LANG: "en_US.UTF-8"
+    # WARNING: this is overridden in the shell.nix, see shell.nix!
     CONFIGURE_ARGS: "--with-intree-gmp"
     TEST_ENV: "x86_64-darwin"
     BUILD_FLAVOUR: "validate"
@@ -503,11 +545,149 @@ validate-x86_64-darwin:
   after_script:
     - cp -Rf $HOME/.cabal cabal-cache
     - .gitlab/ci.sh clean
+
+  # I wish we could just use the nix #! logic, but we can't --run and -i bash
+  # behave very differently. -i bash does not pass any nix related env vars
+  # the whole $stdenv/setup part seems to be missing.
+  script: |
+    set -Eeuo pipefail
+    function runInNixShell() {
+      time nix-shell .gitlab/shell.nix \
+        -I nixpkgs=https://github.com/angerman/nixpkgs/archive/75f7281738b.tar.gz \
+        --argstr system "x86_64-darwin" \
+        --pure \
+        --keep GHC_VERSION --keep CABAL_INSTALL_VERSION --keep BUILD_FLAVOUR \
+        --keep BIN_DIST_PREP_TAR_COMP --keep CPUS --keep PROJECT_DIR \
+        --keep CI_PROJECT_DIR --keep MAKE_ARGS \
+        --keep LANG --keep CONFIGURE_ARGS \
+        --keep MACOSX_DEPLOYMENT_TARGET --keep ac_cv_func_clock_gettime \
+        --run "$1" 2>&1
+    }
+    # fix up config.sub in libraries for the time.
+    # aarch64-darwin is not supported in older config.sub's
+    TIME_START=$(date +%s)
+    find libraries -name config.sub -exec cp config.sub {} \;
+
+    runInNixShell ".gitlab/ci.sh setup" 2>&1
+    TIME_SETUP=$(date +%s)
+    TIME_SETUP_DELTA=$(expr $TIME_SETUP - $TIME_START)
+    echo "Setup took $TIME_SETUP_DELTA seconds"
+
+    runInNixShell ".gitlab/ci.sh configure" 2>&1
+    TIME_CONFIGURE=$(date +%s)
+    TIME_CONFIGURE_DELTA=$(expr $TIME_CONFIGURE - $TIME_SETUP)
+    echo "Configure took $TIME_CONFIGURE_DELTA seconds"
+
+    runInNixShell ".gitlab/ci.sh build_make" 2>&1
+    TIME_BUILD=$(date +%s)
+    TIME_BUILD_DELTA=$(expr $TIME_BUILD - $TIME_CONFIGURE)
+    echo "Build took $TIME_BUILD_DELTA seconds"
+
+    set +e
+    runInNixShell ".gitlab/ci.sh test_make" 2>&1
+    status=$?
+    set -e
+
+    TIME_TEST=$(date +%s)
+    TIME_TEST_DELTA=$(expr $TIME_TEST - $TIME_BUILD)
+    echo "Test took $TIME_TEST_DELTA seconds"
+
+    echo "=== TIMINGS ==="
+    echo "Setup      | $TIME_SETUP_DELTA"
+    echo "Configure  | $TIME_CONFIGURE_DELTA"
+    echo "Build      | $TIME_BUILD_DELTA"
+    echo "Test       | $TIME_TEST_DELTA"
+    exit $status
+
+  artifacts:
+    when: always
+    expire_in: 2 week
+  cache:
+    key: "darwin-x86_64-$GHC_VERSION"
+    paths:
+      - cabal-cache
+      - toolchain
+
+validate-aarch64-darwin:
+  extends: .validate
+  stage: full-build
+  tags:
+    - aarch64-darwin-m1
+
+  # for now make this non mandatory to pass.
+  allow_failure: true
+
+  variables:
+    TEST_TYPE: test
+    MAKE_ARGS: "-Werror"
+    GHC_VERSION: 8.10.3
+    CABAL_INSTALL_VERSION: 3.2.0.0
+    BUILD_FLAVOUR: "perf"
+    BIN_DIST_PREP_TAR_COMP: "ghc-arm64-apple-darwin.tar.xz"
+    # we run on M1's for now, getconf can't be built with nix yet,
+    # and we use a pure shell, so we can't/shouldn't use /usr/bin/getconf
+    # inside th shell.
+    CPUS: 8
+    LANG: "en_US.UTF-8"
+    # WARNING: this is overridden in the shell.nix, see shell.nix!
+    CONFIGURE_ARGS: "--with-intree-gmp"
+
+  # I wish we could just use the nix #! logic, but we can't --run and -i bash
+  # behave very differently. -i bash does not pass any nix related env vars
+  # the whole $stdenv/setup part seems to be missing.
+  script: |
+    set -Eeuo pipefail
+    function runInNixShell() {
+      time nix-shell .gitlab/shell.nix \
+        -I nixpkgs=https://github.com/angerman/nixpkgs/archive/75f7281738b.tar.gz \
+        --argstr system "aarch64-darwin" \
+        --pure \
+        --keep GHC_VERSION --keep CABAL_INSTALL_VERSION --keep BUILD_FLAVOUR \
+        --keep BIN_DIST_PREP_TAR_COMP --keep CPUS --keep PROJECT_DIR \
+        --keep CI_PROJECT_DIR --keep MAKE_ARGS \
+        --keep LANG --keep CONFIGURE_ARGS \
+        --run "$1" 2>&1
+    }
+    # fix up config.sub in libraries for the time.
+    # aarch64-darwin is not supported in older config.sub's
+    TIME_START=$(date +%s)
+    find libraries -name config.sub -exec cp config.sub {} \;
+
+    time runInNixShell ".gitlab/ci.sh setup" 2>&1
+    TIME_SETUP=$(date +%s)
+    TIME_SETUP_DELTA=$(expr $TIME_SETUP - $TIME_START)
+    echo "Setup took $TIME_SETUP_DELTA seconds"
+
+    runInNixShell ".gitlab/ci.sh configure" 2>&1
+    TIME_CONFIGURE=$(date +%s)
+    TIME_CONFIGURE_DELTA=$(expr $TIME_CONFIGURE - $TIME_SETUP)
+    echo "Setup took $TIME_CONFIGURE_DELTA seconds"
+
+    runInNixShell ".gitlab/ci.sh build_make" 2>&1
+    TIME_BUILD=$(date +%s)
+    TIME_BUILD_DELTA=$(expr $TIME_BUILD - $TIME_CONFIGURE)
+    echo "Build took $TIME_BUILD_DELTA seconds"
+
+    set +e
+    runInNixShell ".gitlab/ci.sh test_make" 2>&1
+    status=$?
+    set -e
+    TIME_TEST=$(date +%s)
+    TIME_TEST_DELTA=$(expr $TIME_TEST - $TIME_BUILD)
+    echo "Test took $TIME_TEST_DELTA seconds"
+
+    echo "=== TIMINGS ==="
+    echo "Setup      | $TIME_SETUP_DELTA"
+    echo "Configure  | $TIME_CONFIGURE_DELTA"
+    echo "Build      | $TIME_BUILD_DELTA"
+    echo "Test       | $TIME_TEST_DELTA"
+    exit $status
+
   artifacts:
     when: always
     expire_in: 2 week
   cache:
-    key: "darwin-$GHC_VERSION"
+    key: "darwin-aarch64-$GHC_VERSION"
     paths:
       - cabal-cache
       - toolchain
diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh
index df43491c5b..1b7518c037 100755
--- a/.gitlab/ci.sh
+++ b/.gitlab/ci.sh
@@ -3,9 +3,7 @@
 
 # This is the primary driver of the GitLab CI infrastructure.
 # Run `ci.sh usage` for usage information.
-
-
-set -e -o pipefail
+set -Eeuo pipefail
 
 # Configuration:
 HACKAGE_INDEX_STATE="2020-12-21T14:48:20Z" # TODO dedup with yaml's def
@@ -173,12 +171,18 @@ function show_tool() {
 
 function set_toolchain_paths() {
   needs_toolchain="1"
-  case "$(uname)" in
-    Linux) needs_toolchain="0" ;;
+  case "$(uname -m)-$(uname)" in
+    *-Linux) needs_toolchain="" ;;
     *) ;;
   esac
 
-  if [[ "$needs_toolchain" = "1" ]]; then
+  if [[ -n "${IN_NIX_SHELL:-}" ]]; then
+      needs_toolchain=""
+      GHC="$(which ghc)"
+      CABAL="$(which cabal)"
+      HAPPY="$(which happy)"
+      ALEX="$(which alex)"
+  elif [[ -n "$needs_toolchain" ]]; then
       # These are populated by setup_toolchain
       GHC="$toolchain/bin/ghc$exe"
       CABAL="$toolchain/bin/cabal$exe"
@@ -354,7 +358,7 @@ BUILD_SPHINX_HTML=$BUILD_SPHINX_HTML
 BUILD_SPHINX_PDF=$BUILD_SPHINX_PDF
 BeConservative=YES
 BIGNUM_BACKEND=$BIGNUM_BACKEND
-XZ_CMD=$XZ
+XZ_CMD=${XZ:-}
 
 BuildFlavour=$BUILD_FLAVOUR
 ifneq "\$(BuildFlavour)" ""
@@ -363,7 +367,7 @@ endif
 GhcLibHcOpts+=-haddock
 EOF
 
-  if [ -n "$HADDOCK_HYPERLINKED_SOURCES" ]; then
+  if [ -n "${HADDOCK_HYPERLINKED_SOURCES:-}" ]; then
     echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
   fi
 
@@ -382,7 +386,7 @@ function configure() {
   end_section "booting"
 
   local target_args=""
-  if [[ -n "$target_triple" ]]; then
+  if [[ -n "${target_triple:-}" ]]; then
     target_args="--target=$target_triple"
   fi
 
@@ -390,7 +394,7 @@ function configure() {
   run ./configure \
     --enable-tarballs-autodownload \
     $target_args \
-    $CONFIGURE_ARGS \
+    ${CONFIGURE_ARGS:-} \
     GHC="$GHC" \
     HAPPY="$HAPPY" \
     ALEX="$ALEX" \
@@ -403,10 +407,10 @@ function build_make() {
   if [[ -z "$BIN_DIST_PREP_TAR_COMP" ]]; then
     fail "BIN_DIST_PREP_TAR_COMP is not set"
   fi
-  if [[ -n "$VERBOSE" ]]; then
-    MAKE_ARGS="$MAKE_ARGS V=1"
+  if [[ -n "${VERBOSE:-}" ]]; then
+    MAKE_ARGS="${MAKE_ARGS:-} V=1"
   else
-    MAKE_ARGS="$MAKE_ARGS V=0"
+    MAKE_ARGS="${MAKE_ARGS:-} V=0"
   fi
 
   echo "include mk/flavours/${BUILD_FLAVOUR}.mk" > mk/build.mk
@@ -422,7 +426,7 @@ function fetch_perf_notes() {
 }
 
 function push_perf_notes() {
-  if [ -n "$CROSS_TARGET" ]; then
+  if [ -n "${CROSS_TARGET:-}" ]; then
     info "Can't test cross-compiled build."
     return
   fi
@@ -439,16 +443,16 @@ function determine_metric_baseline() {
 }
 
 function test_make() {
-  if [ -n "$CROSS_TARGET" ]; then
+  if [ -n "${CROSS_TARGET:-}" ]; then
     info "Can't test cross-compiled build."
     return
   fi
 
   run "$MAKE" test_bindist TEST_PREP=YES
-  run "$MAKE" V=0 test \
+  run "$MAKE" V=0 VERBOSE=1 test \
     THREADS="$cores" \
     JUNIT_FILE=../../junit.xml \
-    EXTRA_RUNTEST_OPTS="$RUNTEST_ARGS"
+    EXTRA_RUNTEST_OPTS="${RUNTEST_ARGS:-}"
 }
 
 function build_hadrian() {
@@ -462,7 +466,7 @@ function build_hadrian() {
 }
 
 function test_hadrian() {
-  if [ -n "$CROSS_TARGET" ]; then
+  if [ -n "${CROSS_TARGET:-}" ]; then
     info "Can't test cross-compiled build."
     return
   fi
@@ -476,7 +480,7 @@ function test_hadrian() {
     test \
     --summary-junit=./junit.xml \
     --test-compiler="$TOP"/_build/install/bin/ghc \
-    "runtest.opts+=$RUNTEST_ARGS"
+    "runtest.opts+=${RUNTEST_ARGS:-}"
 }
 
 function cabal_test() {
@@ -517,14 +521,14 @@ function run_hadrian() {
   if [ -z "$BUILD_FLAVOUR" ]; then
     fail "BUILD_FLAVOUR not set"
   fi
-  if [ -z "$BIGNUM_BACKEND" ]; then BIGNUM_BACKEND="gmp"; fi
-  if [ -n "$VERBOSE" ]; then HADRIAN_ARGS="$HADRIAN_ARGS -V"; fi
+  if [ -z "${BIGNUM_BACKEND:-}" ]; then BIGNUM_BACKEND="gmp"; fi
+  if [ -n "${VERBOSE:-}" ]; then HADRIAN_ARGS="${HADRIAN_ARGS:-} -V"; fi
   run hadrian/build-cabal \
     --flavour="$BUILD_FLAVOUR" \
     -j"$cores" \
-    --broken-test="$BROKEN_TESTS" \
+    --broken-test="${BROKEN_TESTS:-}" \
     --bignum=$BIGNUM_BACKEND \
-    $HADRIAN_ARGS \
+    ${HADRIAN_ARGS:-} \
     $@
 }
 
@@ -560,7 +564,7 @@ case "$(uname)" in
   *) fail "uname $(uname) is not supported" ;;
 esac
 
-if [ -n "$CROSS_TARGET" ]; then
+if [ -n "${CROSS_TARGET:-}" ]; then
   info "Cross-compiling for $CROSS_TARGET..."
   target_triple="$CROSS_TARGET"
 fi
diff --git a/.gitlab/shell.nix b/.gitlab/shell.nix
new file mode 100644
index 0000000000..5e32847735
--- /dev/null
+++ b/.gitlab/shell.nix
@@ -0,0 +1,65 @@
+{ system ? "aarch64-darwin"
+#, nixpkgs ? fetchTarball https://github.com/angerman/nixpkgs/archive/257cb120334.tar.gz #apple-silicon.tar.gz
+, pkgs ? import <nixpkgs> { inherit system; }
+, compiler ? if system == "aarch64-darwin" then "ghc8103Binary" else "ghc8103"
+}: pkgs.mkShell {
+  # this prevents nix from trying to write the env-vars file.
+  # we can't really, as NIX_BUILD_TOP/env-vars is not set.
+  noDumpEnvVars=1;
+
+  # stop polluting LDFLAGS with -liconv
+  dontAddExtraLibs = true;
+
+  # we need to inject ncurses into --with-curses-libraries.
+  # the real fix is to teach terminfo to use libcurses on macOS.
+  CONFIGURE_ARGS = "--with-intree-gmp --with-curses-libraries=${pkgs.ncurses.out}/lib";
+
+  # magic speedup pony :facepalm:
+  #
+  # nix has the ugly habbit of duplicating ld flags more than necessary.  This
+  # somewhat consolidates this.
+  shellHook = ''
+  export NIX_LDFLAGS=$(for a in $NIX_LDFLAGS; do echo $a; done |sort|uniq|xargs)
+  export NIX_LDFLAGS_FOR_TARGET=$(for a in $NIX_LDFLAGS_FOR_TARGET; do echo $a; done |sort|uniq|xargs)
+  export NIX_LDFLAGS_FOR_TARGET=$(comm -3 <(for l in $NIX_LDFLAGS_FOR_TARGET; do echo $l; done) <(for l in $NIX_LDFLAGS; do echo $l; done))
+  '';
+
+  nativeBuildInputs = (with pkgs; [
+    haskell.compiler.${compiler}
+    haskell.packages.${compiler}.cabal-install
+    haskell.packages.${compiler}.alex
+    haskell.packages.${compiler}.happy # _1_19_12 is needed for older GHCs.
+
+    clang_11
+    llvm_11
+
+    automake
+    autoconf
+    m4
+
+    gmp
+    ncurses
+    libiconv
+    zlib.out
+    zlib.dev
+    glibcLocales
+    # locale doesn't build yet :-/
+    # locale
+
+    git
+
+    python3
+    # python3Full
+    # python3Packages.sphinx
+    perl
+
+    which
+    wget
+    file
+
+    xz
+    xlibs.lndir
+
+    cacert ])
+  ++ (with pkgs.darwin.apple_sdk.frameworks; [ Foundation Security ]);
+}
diff --git a/boot b/boot
index fbee248d77..ec37c169ad 100755
--- a/boot
+++ b/boot
@@ -8,6 +8,7 @@ import argparse
 from textwrap import dedent
 import subprocess
 import re
+import shutil
 
 cwd = os.getcwd()
 
@@ -137,6 +138,8 @@ def autoreconf():
     for dir_ in ['.'] + glob.glob('libraries/*/'):
         if os.path.isfile(os.path.join(dir_, 'configure.ac')):
             print("Booting %s" % dir_)
+            if dir_ != '.':
+                shutil.copyfile('config.sub', os.path.join(dir_, 'config.sub'))
             processes[dir_] = subprocess.Popen(['sh', '-c', reconf_cmd], cwd=dir_)
 
     # Wait for all child processes to finish.
diff --git a/rules/library-path.mk b/rules/library-path.mk
index 9a398eebf0..92511f2c94 100644
--- a/rules/library-path.mk
+++ b/rules/library-path.mk
@@ -13,9 +13,8 @@
 # $1 = paths to prepend
 ifeq "$(TargetOS_CPP)" "mingw32"
 prependLibraryPath = $(error Do not know how to prependLibraryPath on Windows)
-else ifeq "$(TargetOS_CPP)" "darwin"
-prependLibraryPath = export DYLD_LIBRARY_PATH="$1$${DYLD_LIBRARY_PATH:+:$$DYLD_LIBRARY_PATH}"
-else
+else ifneq "$(TargetOS_CPP)" "darwin"
 prependLibraryPath = export LD_LIBRARY_PATH="$1$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}"
+else
+prependLibraryPath =
 endif
-
diff --git a/testsuite/driver/cpu_features.py b/testsuite/driver/cpu_features.py
index 7b4340b61c..6e32d3b445 100644
--- a/testsuite/driver/cpu_features.py
+++ b/testsuite/driver/cpu_features.py
@@ -29,7 +29,9 @@ def get_cpu_features():
         return flags
 
     elif config.os == 'darwin':
-        out = subprocess.check_output(['sysctl', 'hw']).decode('UTF-8')
+        # we hardcode the sysctl path, otherwise we rely on /usr/sbin being in
+        # path.
+        out = subprocess.check_output(['/usr/sbin/sysctl', 'hw']).decode('UTF-8')
         features = set()
         def check_feature(darwin_name, our_name=None):
             if re.search(r'hw\.optional.%s:\s*1' % darwin_name, out) is not None:
@@ -66,4 +68,3 @@ if __name__ == '__main__':
     import sys
     config.os = sys.argv[1]
     print(get_cpu_features())
-
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index d4425b5da5..ac0d4f7100 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -266,7 +266,7 @@ def format_path(path):
 
 # On Windows we need to set $PATH to include the paths to all the DLLs
 # in order for the dynamic library tests to work.
-if windows or darwin:
+if windows:
     pkginfo = getStdout([config.ghc_pkg, 'dump'])
     topdir = config.libdir
     if windows:
@@ -284,12 +284,9 @@ if windows or darwin:
             if path.startswith('"'):
                 path = re.sub('^"(.*)"$', '\\1', path)
                 path = re.sub('\\\\(.)', '\\1', path)
-            if windows:
+
                 path = format_path(path)
                 ghc_env['PATH'] = os.pathsep.join([path, ghc_env.get("PATH", "")])
-            else:
-                # darwin
-                ghc_env['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, ghc_env.get("DYLD_LIBRARY_PATH", "")])
 
 testopts_local.x = TestOptions()
 
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index b5d2744e58..3c82a7de2f 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1809,9 +1809,10 @@ def interpreter_run(name: TestName,
 
     # check the exit code
     if exit_code != getTestOpts().exit_code:
-        print('Wrong exit code for ' + name + '(' + way + ') (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
-        dump_stdout(name)
-        dump_stderr(name)
+        if config.verbose >= 1 and _expect_pass(way):
+            print('Wrong exit code for ' + name + '(' + way + ') (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
+            dump_stdout(name)
+            dump_stderr(name)
         message = format_bad_exit_code_message(exit_code)
         return failBecause(message,
                            stderr=read_stderr(name),
@@ -2236,9 +2237,14 @@ def normalise_errmsg(s: str) -> str:
     # and not understood by older binutils (ar, ranlib, ...)
     s = modify_lines(s, lambda l: re.sub('^(.+)warning: (.+): unsupported GNU_PROPERTY_TYPE \(5\) type: 0xc000000(.*)$', '', l))
 
-    # filter out nix garbage, that just keeps on showing up as errors on darwin
-    s = modify_lines(s, lambda l: re.sub('^(.+)\.dylib, ignoring unexpected dylib file$','', l))
-
+    s = re.sub('ld: warning: passed .* min versions \(.*\) for platform macOS. Using [\.0-9]+.','',s)
+    s = re.sub('ld: warning: -sdk_version and -platform_version are not compatible, ignoring -sdk_version','',s)
+    # ignore superfluous dylibs passed to the linker.
+    s = re.sub('ld: warning: .*, ignoring unexpected dylib file\n','',s)
+    # ignore LLVM Version mismatch garbage; this will just break tests.
+    s = re.sub('You are using an unsupported version of LLVM!.*\n','',s)
+    s = re.sub('Currently only [\.0-9]+ is supported. System LLVM version: [\.0-9]+.*\n','',s)
+    s = re.sub('We will try though\.\.\..*\n','',s)
     return s
 
 # normalise a .prof file, so that we can reasonably compare it against
@@ -2312,6 +2318,15 @@ def normalise_output( s: str ) -> str:
     # ghci outputs are pretty unstable with -fexternal-dynamic-refs, which is
     # requires for -fPIC
     s = re.sub('  -fexternal-dynamic-refs\n','',s)
+    s = re.sub('ld: warning: passed .* min versions \(.*\) for platform macOS. Using [\.0-9]+.','',s)
+    s = re.sub('ld: warning: -sdk_version and -platform_version are not compatible, ignoring -sdk_version','',s)
+    # ignore superfluous dylibs passed to the linker.
+    s = re.sub('ld: warning: .*, ignoring unexpected dylib file\n','',s)
+    # ignore LLVM Version mismatch garbage; this will just break tests.
+    s = re.sub('You are using an unsupported version of LLVM!.*\n','',s)
+    s = re.sub('Currently only [\.0-9]+ is supported. System LLVM version: [\.0-9]+.*\n','',s)
+    s = re.sub('We will try though\.\.\..*\n','',s)
+
     return s
 
 def normalise_asm( s: str ) -> str:
diff --git a/testsuite/tests/concurrent/should_run/conc059_c.c b/testsuite/tests/concurrent/should_run/conc059_c.c
index 58cb595073..89532caf4c 100644
--- a/testsuite/tests/concurrent/should_run/conc059_c.c
+++ b/testsuite/tests/concurrent/should_run/conc059_c.c
@@ -5,6 +5,8 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <stdio.h>
+// stdlib is needed for exit()
+#include <stdlib.h>
 #if mingw32_HOST_OS
 #include <windows.h>
 #else
diff --git a/testsuite/tests/driver/all.T b/testsuite/tests/driver/all.T
index 94ecb3006c..ad2a477f74 100644
--- a/testsuite/tests/driver/all.T
+++ b/testsuite/tests/driver/all.T
@@ -121,7 +121,8 @@ else:
   only_darwin = skip
 
 test('static001', [extra_files(['Static001.hs']),
-                   only_darwin, expect_broken(8127)],
+                   only_darwin,
+                   when(arch('x86_64'), expect_broken(8127))],
      makefile_test, ['static001'])
 
 test('dynHelloWorld',
@@ -260,8 +261,12 @@ test('T12955', normal, makefile_test, [])
 
 test('T12971', [when(opsys('mingw32'), fragile(17945)), ignore_stdout], makefile_test, [])
 test('json', normal, compile_fail, ['-ddump-json'])
-test('json2', normalise_version('base','ghc-prim'), compile, ['-ddump-types -ddump-json'])
-test('T16167', exit_code(1), run_command, 
+
+# json2 test is sensitive to the LLVM not supported ouput from GHC. ANd the error
+# won't tell.  It looks unrelated and is annoying to debug. Hence we disable the
+# warning to prevent spurious errors.
+test('json2', normalise_version('base','ghc-prim'), compile, ['-ddump-types -ddump-json -Wno-unsupported-llvm-version'])
+test('T16167', exit_code(1), run_command,
      ['{compiler} -x hs -e ":set prog T16167.hs" -ddump-json T16167.hs'])
 test('T13604', [], makefile_test, [])
 test('T13604a', [], makefile_test, [])
diff --git a/testsuite/tests/ghci/linking/all.T b/testsuite/tests/ghci/linking/all.T
index 29a314238c..82a7822ea4 100644
--- a/testsuite/tests/ghci/linking/all.T
+++ b/testsuite/tests/ghci/linking/all.T
@@ -11,11 +11,12 @@ test('ghcilink002', [extra_files(['TestLink.hs', 'f.c']),
      makefile_test, ['ghcilink002'])
 
 test('ghcilink003',
-     [unless(doing_ghci, skip),
-      # libstdc++ is named differently on FreeBSD
-      when(opsys('freebsd'), expect_broken(17739))],
-     makefile_test,
-     ['ghcilink003'])
+     [ unless(doing_ghci, skip),
+       # libstdc++ is named differently on FreeBSD
+       when(opsys('freebsd'), expect_broken(17739)),
+       when(opsys('darwin'), fragile(16083))
+     ], makefile_test, ['ghcilink003'])
+
 
 test('ghcilink004',
      [extra_files(['TestLink.hs', 'f.c']),
@@ -31,11 +32,11 @@ test('ghcilink005',
      makefile_test, ['ghcilink005'])
 
 test('ghcilink006',
-     [unless(doing_ghci, skip),
-      # libstdc++ is named differently on FreeBSD
-      when(opsys('freebsd'), expect_broken(17739))],
-     makefile_test,
-     ['ghcilink006'])
+     [ unless(doing_ghci, skip),
+       # libstdc++ is named differently on FreeBSD
+       when(opsys('freebsd'), expect_broken(17739)),
+       when(opsys('darwin'), fragile(16083))
+     ], makefile_test, ['ghcilink006'])
 
 test('T3333',
      [unless(doing_ghci, skip),
diff --git a/testsuite/tests/ghci/scripts/all.T b/testsuite/tests/ghci/scripts/all.T
index 64f87bc7e2..b3b2b20e60 100755
--- a/testsuite/tests/ghci/scripts/all.T
+++ b/testsuite/tests/ghci/scripts/all.T
@@ -328,4 +328,3 @@ test('T19197', normal, ghci_script, ['T19197.script'])
 test('T19158', normal, ghci_script, ['T19158.script'])
 test('T19279', normal, ghci_script, ['T19279.script'])
 test('T19310', normal, ghci_script, ['T19310.script'])
-
diff --git a/testsuite/tests/rts/all.T b/testsuite/tests/rts/all.T
index 848314928a..085e4b1f12 100644
--- a/testsuite/tests/rts/all.T
+++ b/testsuite/tests/rts/all.T
@@ -53,6 +53,11 @@ test('divbyzero',
       # behavior on division-by-zero (#10332).
       omit_ways(llvm_ways),
       when(not(have_ncg()), skip),
+      # Aarch64 does not have div-by-zero exceptions for sdiv/udiv.
+      # The only option would be to implement this in assembly with checks for
+      # each devision. Neither gcc, nor llvm do this as of right now.  Microsoft
+      # apparently does so though?
+      when(arch('aarch64'), skip),
       # Apparently the output can be different on different
       # Linux setups, so just ignore it. As long as we get
       # the right exit code we're OK.
diff --git a/testsuite/tests/rts/linker/all.T b/testsuite/tests/rts/linker/all.T
index e89a246165..7bf1d54762 100644
--- a/testsuite/tests/rts/linker/all.T
+++ b/testsuite/tests/rts/linker/all.T
@@ -12,7 +12,7 @@ test('unsigned_reloc_macho_x64',
 test('section_alignment',
      [
          extra_files(['runner.c', 'section_alignment.c']),
-         unless(opsys('darwin') and arch('x86_64'), expect_broken(13624))
+         unless(opsys('darwin'), expect_broken(13624))
      ],
      run_command, ['$MAKE -s --no-print-directory section_alignment'])