Windows: Update tarballs to GCC 9.2 and remove MAX_PATH limit.

2 files changed, 126 insertions, 61 deletions

diff --git a/mk/get-win32-tarballs.sh b/mk/get-win32-tarballs.sh
index fd47232b91..4732ede52d 100755
--- a/mk/get-win32-tarballs.sh
+++ b/mk/get-win32-tarballs.sh
@@ -2,6 +2,7 @@
 
 tarball_dir='ghc-tarballs'
 missing_files=0
+pkg_variant="phyx"
 
 # see #12502
 if test -z "$FIND"; then FIND="find"; fi
@@ -37,7 +38,7 @@ download_file() {
             return
         else
             echo "Downloading ${description} to ${dest_dir}..."
-            $curl_cmd || echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk || {
+            $curl_cmd || (echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk) || {
                 rm -f "${dest_file}"
                 fail "ERROR: Download failed."
                 exit 1
@@ -51,11 +52,11 @@ download_file() {
         echo "Downloading ${description} (signature) to ${dest_dir}..."
         local curl_cmd="curl -f -L ${file_url}.sig -o ${sig_file} --create-dirs -# ${extra_curl_opts}"
         if test -n "${backup_url}"; then
-            local curl_cmd_bnk="curl -f -L ${backup_url} -o ${sig_file} --create-dirs -# ${extra_curl_opts}"
+            local curl_cmd_bnk="curl -f -L "${backup_url}.sig" -o ${sig_file} --create-dirs -# ${extra_curl_opts}"
         else
             local curl_cmd_bnk="true"
         fi
-        $curl_cmd || echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk || {
+        $curl_cmd || (echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk) || {
                 rm -f "${dest_file}.sig"
                 fail "ERROR: Download failed."
                 exit 1
@@ -101,26 +102,23 @@ download_tarballs() {
     local package_prefix="mingw-w64"
     local format_url="/${mingw_arch}/${package_prefix}-${mingw_arch}"
 
-    download_mingw "${format_url}-crt-git-5.0.0.4795.e3d96cb1-1-any.pkg.tar.xz"
-    download_mingw "${format_url}-winpthreads-git-5.0.0.4850.d1662dc7-1-any.pkg.tar.xz"
-    download_mingw "${format_url}-headers-git-5.0.0.4966.1eee2140-1-any.pkg.tar.xz"
-    download_mingw "${format_url}-libwinpthread-git-5.0.0.4850.d1662dc7-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-crt-git-7.0.0.5491.fe45801e-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-winpthreads-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-headers-git-7.0.0.5490.9ec54ed1-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-libwinpthread-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz"
     download_mingw "${format_url}-zlib-1.2.8-9-any.pkg.tar.xz"
-    download_mingw "${format_url}-isl-0.18-1-any.pkg.tar.xz"
-    download_mingw "${format_url}-mpfr-3.1.6-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-isl-0.21-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-mpfr-4.0.2-2-any.pkg.tar.xz"
     download_mingw "${format_url}-gmp-6.1.2-1-any.pkg.tar.xz"
-    download_mingw "${format_url}-binutils-2.29.1-1-any.pkg.tar.xz"
-    download_mingw "${format_url}-libidn2-2.0.4-1-any.pkg.tar.xz"
-    download_mingw "${format_url}-gcc-7.2.0-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-binutils-2.32-3-$pkg_variant.pkg.tar.xz"
+    download_mingw "${format_url}-libidn2-2.2.0-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-gcc-9.2.0-1-$pkg_variant.pkg.tar.xz"
+    download_mingw "${format_url}-mpc-1.1.0-1-any.pkg.tar.xz"
+    download_mingw "${format_url}-windows-default-manifest-6.4-3-any.pkg.tar.xz"
 
     # Upstream is unfortunately quite inconsistent in naming
     if test "$mingw_arch" != "sources"; then
-        download_mingw "${format_url}-mpc-1.0.3-2-any.pkg.tar.xz"
-        download_mingw "${format_url}-gcc-libs-7.2.0-1-any.pkg.tar.xz"
-    else
-        local format_url="${mingw_base_url}/${mingw_arch}/${package_prefix}"
-        download_mingw "${format_url}-i686-mpc-1.0.3-2.src.tar.gz"
-        download_mingw "${format_url}-x86_64-mpc-1.0.3-2.src.tar.gz"
+        download_mingw "${format_url}-gcc-libs-9.2.0-1-$pkg_variant.pkg.tar.xz"
     fi
 
     if ! test "$missing_files" = "0"
@@ -131,7 +129,7 @@ download_tarballs() {
 
 download_i386() {
     mingw_arch="i686"
-    tarball_dest_dir="mingw-w64/x86"
+    tarball_dest_dir="mingw-w64/i686"
     download_tarballs
 }
 
@@ -171,6 +169,54 @@ sync_binaries_and_sources() {
     done
 }
 
+patch_single_file () {
+    local patcher_base="$1"
+    local filename=$(readlink -f "$2")
+    local filepath=$(dirname "$filename")
+    local patcher="$patcher_base/iat-patcher.exe"
+    $patcher install "$filename" > /dev/null
+    rm -f "$filename.bak"
+    for file in $patcher_base/*.dll; do cp -f "$file" "${filepath}"; done
+    echo "Patched '$filename'"
+}
+
+patch_tarball () {
+    local tarball_name="$1"
+    local filename=$(basename "$tarball_name")
+    local filepath=$(dirname "$tarball_name")
+    local newfile=`echo "$filepath/$filename" | sed -e 's/-any/-phyx/'`
+    local arch=""
+
+    echo "=> ${filename}"
+
+    case $1 in
+        *x86_64*)
+            arch="x86_64"
+            ;;
+        *i686*)
+            arch="i686"
+            ;;
+        *)
+        echo "unknown architecture detected.  Stopping."
+        exit 1
+        ;;
+    esac
+
+    local base="$(pwd)"
+    local patcher_base="$(pwd)/ghc-tarballs/ghc-jailbreak/$arch"
+    local tmpdir="ghc-tarballs/tmpdir"
+    mkdir -p $tmpdir
+    cd $tmpdir
+    tar xJf "$base/$tarball_name"
+    find . -iname "*.exe" -exec bash -c \
+      'patch_single_file "'"${patcher_base}"'" "$0"' {} \;
+    tar cJf "$base/$newfile" .
+    cd "$base"
+    rm -rf $tmpdir
+    gpg --output "$base/${newfile}.sig" --detach-sig "$base/$newfile"
+    rm -f "$base/$tarball_name"
+}
+
 show_hashes_for_binaries() {
     $FIND ghc-tarballs/ -iname "*.*" | xargs md5sum | grep -v "\.sig" | sed -s "s/\*//"
 }
@@ -182,11 +228,13 @@ usage() {
     echo
     echo "Where <action> is one of,"
     echo ""
-    echo "    download     download the necessary tarballs for the given architecture"
-    echo "    fetch        download the necessary tarballs for the given architecture but doesn't verify their md5."
-    echo "    verify       verify the existence and correctness of the necessary tarballs"
-    echo "    hash         generate md5 hashes for inclusion in win32-tarballs.md5sum"
-    echo "    sync         upload packages downloaded with 'fetch mirror' to haskell.org"
+    echo "    download   download the necessary tarballs for the given architecture"
+    echo "    fetch      download the necessary tarballs for the given architecture but doesn't verify their md5."
+    echo "    grab       download the necessary tarballs using patched toolchains for the given architecture but doesn't verify their md5."
+    echo "    verify     verify the existence and correctness of the necessary tarballs"
+    echo "    patch      jailbreak the binaries in the tarballs and remove MAX_PATH limitations."
+    echo "    hash       generate md5 hashes for inclusion in win32-tarballs.md5sum"
+    echo "    sync       upload packages downloaded with 'fetch mirror' to haskell.org"
     echo ""
     echo "and <arch> is one of i386, x86_64,all or mirror (which includes sources)"
 }
@@ -199,7 +247,12 @@ case $1 in
         ;;
     fetch)
         download=1
+        verify=
+        ;;
+    grab)
+        download=1
         verify=0
+        pkg_variant="any"
         ;;
     verify)
         download=0
@@ -214,6 +267,28 @@ case $1 in
         show_hashes_for_binaries
         exit 1
         ;;
+    # This routine will download the latest ghc-jailbreak and unpack binutils and
+    # the ghc tarballs and patches every .exe in each.  Along with this is copies
+    # two dlls in every folder that it patches a .exe in.  Afterwards it re-creates
+    # the tarballs and generates a new signature file.
+    patch)
+        export -f patch_tarball
+        export -f patch_single_file
+
+        echo "Downloading ghc-jailbreak..."
+        curl -f -L https://mistuke.blob.core.windows.net/binaries/ghc-jailbreak-0.3.tar.gz \
+            -o ghc-tarballs/ghc-jailbreak/ghc-jailbreak.tar.gz --create-dirs -#
+        tar -C ghc-tarballs/ghc-jailbreak/ -xf ghc-tarballs/ghc-jailbreak/ghc-jailbreak.tar.gz
+
+        find ghc-tarballs/mingw-w64/ \(   -iname "*binutils*.tar.xz" \
+                                       -o -iname "*gcc*.tar.xz" \) \
+        -exec bash -c 'patch_tarball "$0"' {} \;
+
+        rm -rf ghc-tarballs/ghc-jailbreak
+
+        echo "Finished tarball generation, toolchain has been pre-patched."
+        exit 0
+        ;;
     *)
         usage
         exit 1
@@ -236,6 +311,7 @@ case $2 in
         download_i386
         download_x86_64
         verify=0
+        sigs=0
         download_sources
         show_hashes_for_binaries
         ;;
diff --git a/mk/win32-tarballs.md5sum b/mk/win32-tarballs.md5sum
index 2c2d3df93a..5d23609de5 100644
--- a/mk/win32-tarballs.md5sum
+++ b/mk/win32-tarballs.md5sum
@@ -1,39 +1,28 @@
-d0b3537c3f56b6f396b3dc23a905d186 ghc-tarballs/mingw-w64/sources/mingw-w64-crt-git-5.0.0.4795.e3d96cb1-1.src.tar.gz
-51d1006727d50a599bf30c34537bb6a9 ghc-tarballs/mingw-w64/sources/mingw-w64-binutils-2.29.1-1.src.tar.gz
-4578c232f7ed327bd6a364737cb018f5 ghc-tarballs/mingw-w64/sources/mingw-w64-gcc-7.2.0-1.src.tar.gz
-b605210c3d1b03092e3b223d1ccf4001 ghc-tarballs/mingw-w64/sources/mingw-w64-gmp-6.1.2-1.src.tar.gz
-1ad83604bb58ecf43f26843975046c1c ghc-tarballs/mingw-w64/sources/mingw-w64-headers-git-5.0.0.4966.1eee2140-1.src.tar.gz
-5c24d319656ed61e517dfa0fee7e475c ghc-tarballs/mingw-w64/sources/mingw-w64-i686-mpc-1.0.3-2.src.tar.gz
-214032557d2474f10c23c33bdaced8e5 ghc-tarballs/mingw-w64/sources/mingw-w64-isl-0.18-1.src.tar.gz
-8c7efe60059c4c80d0d1966c943b5aaa ghc-tarballs/mingw-w64/sources/mingw-w64-libidn2-2.0.4-1.src.tar.gz
-18e9d17e968773c0691c50d9ceeede34 ghc-tarballs/mingw-w64/sources/mingw-w64-mpfr-3.1.6-1.src.tar.gz
-0462bc73a8b15a68dece429f6f33ca81 ghc-tarballs/mingw-w64/sources/mingw-w64-winpthreads-git-5.0.0.4850.d1662dc7-1.src.tar.gz
-d4261042f502998006c57f0e6517998e ghc-tarballs/mingw-w64/sources/mingw-w64-x86_64-mpc-1.0.3-2.src.tar.gz
-534bb4756482f3271308576cdadfe5dc ghc-tarballs/mingw-w64/x86/mingw-w64-i686-crt-git-5.0.0.4795.e3d96cb1-1-any.pkg.tar.xz
-32078e2a67e34ff6c0b1795fe6ed9d36 ghc-tarballs/mingw-w64/sources/mingw-w64-zlib-1.2.8-9.src.tar.gz
-66e4fc3e62c0fa6723c0d27bdb832962 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-binutils-2.29.1-1-any.pkg.tar.xz
-634ad635913713abc0dbcb0ebef8c85b ghc-tarballs/mingw-w64/x86/mingw-w64-i686-gcc-7.2.0-1-any.pkg.tar.xz
-5343540bc6caa7ada564483f3a1fdff7 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-gcc-libs-7.2.0-1-any.pkg.tar.xz
-2f97e71e4ec122151c20350433268d8b ghc-tarballs/mingw-w64/x86/mingw-w64-i686-gmp-6.1.2-1-any.pkg.tar.xz
-07f374e6eb4846169921551ee73103e6 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-headers-git-5.0.0.4966.1eee2140-1-any.pkg.tar.xz
-ac7017bd5d69030cfe76e475e213b43a ghc-tarballs/mingw-w64/x86/mingw-w64-i686-isl-0.18-1-any.pkg.tar.xz
-f758e0d121619b166dbe36060c641767 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-libidn2-2.0.4-1-any.pkg.tar.xz
-3c718f86311bdd8c4438513a0f417391 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-libwinpthread-git-5.0.0.4850.d1662dc7-1-any.pkg.tar.xz
-719e76fa7a54a8676d2e60af3bb13c45 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-mpc-1.0.3-2-any.pkg.tar.xz
-3780a25a6f20eef9b143f47f4b615e39 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-crt-git-5.0.0.4795.e3d96cb1-1-any.pkg.tar.xz
-9d71b5b4698310a6b961a7d9e273a204 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-mpfr-3.1.6-1-any.pkg.tar.xz
-8912539507692ecf3cebcd5405e1a526 ghc-tarballs/mingw-w64/x86/mingw-w64-i686-winpthreads-git-5.0.0.4850.d1662dc7-1-any.pkg.tar.xz
-87c65e9b2930436a75dfd7d459ae98cb ghc-tarballs/mingw-w64/x86/mingw-w64-i686-zlib-1.2.8-9-any.pkg.tar.xz
-a7e26551a47ce247e432d3f3db416f2b ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-binutils-2.29.1-1-any.pkg.tar.xz
-c4ce9e9a696a76a350d6ed4222fe205e ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-gcc-7.2.0-1-any.pkg.tar.xz
-872a54d953e7d9e6f375e3712d540701 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-gcc-libs-7.2.0-1-any.pkg.tar.xz
+8af4600c30671775a1dde3b60fe75854 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-binutils-2.32-3-phyx.pkg.tar.xz
+c442f8999c10a24059a79a0a8974bc46 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-crt-git-7.0.0.5491.fe45801e-1-any.pkg.tar.xz
+dddc20e7431ab369f6e2726feb4f65d1 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-gcc-9.2.0-1-phyx.pkg.tar.xz
+64130e9d870de2292d4627a67fabee51 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-gcc-libs-9.2.0-1-phyx.pkg.tar.xz
+2f97e71e4ec122151c20350433268d8b ghc-tarballs/mingw-w64/i686/mingw-w64-i686-gmp-6.1.2-1-any.pkg.tar.xz
+192a25fe284cf8a78851a673d6ef672a ghc-tarballs/mingw-w64/i686/mingw-w64-i686-headers-git-7.0.0.5490.9ec54ed1-1-any.pkg.tar.xz
+a18983513ec46ff0716dc5742604a78b ghc-tarballs/mingw-w64/i686/mingw-w64-i686-isl-0.21-1-any.pkg.tar.xz
+c8dabb32fabe2492878e070ffad473f0 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-libidn2-2.2.0-1-any.pkg.tar.xz
+d05ca6e90126c17db19a8cd32e7347e8 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-libwinpthread-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz
+cae596bd144ad70875ef8f21e5bdb77d ghc-tarballs/mingw-w64/i686/mingw-w64-i686-mpc-1.1.0-1-any.pkg.tar.xz
+317a7a490da5c88f63638fbc5461b51e ghc-tarballs/mingw-w64/i686/mingw-w64-i686-mpfr-4.0.2-2-any.pkg.tar.xz
+15a5557d7b321bb26436dcf7adced5b7 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-windows-default-manifest-6.4-3-any.pkg.tar.xz
+beb76cd6141d11c000a1f5ff2ad34971 ghc-tarballs/mingw-w64/i686/mingw-w64-i686-winpthreads-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz
+87c65e9b2930436a75dfd7d459ae98cb ghc-tarballs/mingw-w64/i686/mingw-w64-i686-zlib-1.2.8-9-any.pkg.tar.xz
+4dca7def5591b8f999e5fa20084cc7a9 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-binutils-2.32-3-phyx.pkg.tar.xz
+d4336ca77b5edf3126a6c1358952567a ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-crt-git-7.0.0.5491.fe45801e-1-any.pkg.tar.xz
+e84e0426204b73af13752b65a035dfc2 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-gcc-9.2.0-1-phyx.pkg.tar.xz
+b6200865004cb558c8f5cd7526975602 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-gcc-libs-9.2.0-1-phyx.pkg.tar.xz
 1e1f79abc9fc0534d360c2889b10a9f5 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-gmp-6.1.2-1-any.pkg.tar.xz
-8cce0779e70d5c3ef3564a11fb3bd931 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-headers-git-5.0.0.4966.1eee2140-1-any.pkg.tar.xz
-e1c25a67656e3667847be1f5eba73862 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-isl-0.18-1-any.pkg.tar.xz
-6faaa722dce8af954b30e39c4213d657 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-libidn2-2.0.4-1-any.pkg.tar.xz
-df50f51f32efe8bff7a87f26c6832045 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-libwinpthread-git-5.0.0.4850.d1662dc7-1-any.pkg.tar.xz
-df1a7d4050568d83c265ae78c32ef30b ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-mpc-1.0.3-2-any.pkg.tar.xz
-d15495e807e2a87b917febedbfb3f5b3 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-mpfr-3.1.6-1-any.pkg.tar.xz
-c5ffd1a27eb459242b46d76b066e557e ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-winpthreads-git-5.0.0.4850.d1662dc7-1-any.pkg.tar.xz
+ef6ad2b29b92264c78d7ba6f0a875838 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-headers-git-7.0.0.5490.9ec54ed1-1-any.pkg.tar.xz
+623c059a9691ee7fc74b9e3f89b35782 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-isl-0.21-1-any.pkg.tar.xz
+29a39230c6208366b9d046fbe5bdde6f ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-libidn2-2.2.0-1-any.pkg.tar.xz
+9953a82fd713f0c7ece69d9228238f4d ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-libwinpthread-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz
+c3aa7f45926edaaafe3bba988e327271 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-mpc-1.1.0-1-any.pkg.tar.xz
+951d8046e65768e00b6dfe1bb6e23016 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-mpfr-4.0.2-2-any.pkg.tar.xz
+8360a1dd2f6f4fd518907530f0839a48 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-windows-default-manifest-6.4-3-any.pkg.tar.xz
+e7cf8d4ac9e9b3f79c16a09413dd1322 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-winpthreads-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz
 60c3a388478f411b7a0908441ebeb537 ghc-tarballs/mingw-w64/x86_64/mingw-w64-x86_64-zlib-1.2.8-9-any.pkg.tar.xz
-b21d1681b61cf7a024e854096285b02e ghc-tarballs/perl/ghc-perl-1.tar.gz