summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBen Gamari <ben@smart-cactus.org>2020-01-19 15:04:30 -0500
committerBen Gamari <ben@smart-cactus.org>2020-02-14 10:16:36 -0500
commit0725f4bbc7f59282ee5fe41619099957030d85ff (patch)
tree4fcfac2cdac250ac2d6db89e7343fbac62317119
parent813842f42023457ccb4d93b5906e0f492e4721b2 (diff)
downloadhaskell-0725f4bbc7f59282ee5fe41619099957030d85ff.tar.gz
Rework handling of win32 toolchain tarballs
-rw-r--r--.gitlab-ci.yml2
-rw-r--r--configure.ac4
-rw-r--r--distrib/configure.ac.in6
-rw-r--r--hadrian/src/Rules/Configure.hs3
-rwxr-xr-xmk/get-win32-tarballs.py59
-rwxr-xr-xmk/get-win32-tarballs.sh326
6 files changed, 64 insertions, 336 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 63c6983a38..292ff7db96 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1034,7 +1034,7 @@ source-tarball:
- ghc-*.tar.xz
- version
script:
- - mk/get-win32-tarballs.sh download all
+ - python3 mk/get-win32-tarballs.py download all
- ./boot
- ./configure
- make sdist
diff --git a/configure.ac b/configure.ac
index 6ea9a82395..da0eb23fad 100644
--- a/configure.ac
+++ b/configure.ac
@@ -347,7 +347,7 @@ set_up_tarballs() {
else
action="download"
fi
- mk/get-win32-tarballs.sh $action $HostArch > missing-win32-tarballs
+ $PYTHON mk/get-win32-tarballs.py $action $mingw_arch > missing-win32-tarballs
case $? in
0)
rm missing-win32-tarballs
@@ -359,7 +359,7 @@ set_up_tarballs() {
echo
echo " * run configure with the --enable-tarballs-autodownload option"
echo
- echo " * run mk/get-win32-tarballs.sh download ${HostArch}"
+ echo " * run mk/get-win32-tarballs.py download $mingw_arch"
echo
echo " * manually download the files listed in ./missing-win32-tarballs and place"
echo " them in the ghc-tarballs directory."
diff --git a/distrib/configure.ac.in b/distrib/configure.ac.in
index c3df6203b5..bfff387ee8 100644
--- a/distrib/configure.ac.in
+++ b/distrib/configure.ac.in
@@ -62,12 +62,6 @@ dnl ** figure out how to do a BSD-ish install **
AC_PROG_INSTALL
#
-dnl ** find `find`, used by get-win32-tarballs.sh (see #12502)
-#
-FP_PROG_FIND()
-export FIND
-
-#
dnl ** how to do symlinks **
#
AC_PROG_LN_S()
diff --git a/hadrian/src/Rules/Configure.hs b/hadrian/src/Rules/Configure.hs
index 931801be5d..f904475ddf 100644
--- a/hadrian/src/Rules/Configure.hs
+++ b/hadrian/src/Rules/Configure.hs
@@ -33,7 +33,8 @@ configureRules = do
-- We cannot use windowsHost here due to a cyclic dependency.
when windowsHost $ do
putBuild "| Checking for Windows tarballs..."
- quietly $ cmd ["bash", "mk/get-win32-tarballs.sh", "download", System.arch]
+ pythonPath <- builderPath Python
+ quietly $ cmd [pythonPath, "mk/get-win32-tarballs.py", "download", System.arch]
let srcs = map (<.> "in") outs
context = vanillaContext Stage0 compiler
need srcs
diff --git a/mk/get-win32-tarballs.py b/mk/get-win32-tarballs.py
new file mode 100755
index 0000000000..3c4baf01e7
--- /dev/null
+++ b/mk/get-win32-tarballs.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pathlib import Path
+import urllib.request
+import subprocess
+import argparse
+
+TARBALL_VERSION = '0.1'
+BASE_URL = "https://downloads.haskell.org/ghc/mingw/{}".format(TARBALL_VERSION)
+BASE_URL = "http://home.smart-cactus.org/~ben/ghc/mingw/{}".format(TARBALL_VERSION)
+DEST = Path('ghc-tarballs/mingw-w64')
+ARCHS = ['i686', 'x86_64', 'sources']
+
+def file_url(arch: str, fname: str) -> str:
+ return "{base}/{arch}/{fname}".format(
+ base=BASE_URL,
+ arch=arch,
+ fname=fname)
+
+def fetch(url: str, dest: Path):
+ print('Fetching', url, '=>', dest)
+ urllib.request.urlretrieve(url, dest)
+
+def fetch_arch(arch: str):
+ req = urllib.request.urlopen(file_url(arch, 'MANIFEST'))
+ files = req.read().decode('UTF-8').split('\n')
+ d = DEST / arch
+ if not d.is_dir():
+ d.mkdir(parents=True)
+ fetch(file_url(arch, 'SHA256SUMS'), d / 'SHA256SUMS')
+ for fname in files:
+ if not (d / fname).is_file():
+ fetch(file_url(arch, fname), d / fname)
+
+ verify(arch)
+
+def verify(arch: str):
+ cmd = ['sha256sum', '--quiet', '--check', '--ignore-missing', 'SHA256SUMS']
+ subprocess.check_call(cmd, cwd=DEST / arch)
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument('mode', choices=['verify', 'download'])
+ parser.add_argument(
+ 'arch',
+ choices=ARCHS + ['all'],
+ help="Architecture to fetch (either i686, x86_64, sources, or all)")
+ args = parser.parse_args()
+
+ action = fetch_arch if args.mode == 'download' else verify
+ if args.arch == 'all':
+ for arch in ARCHS:
+ action(arch)
+ else:
+ action(args.arch)
+
+if __name__ == '__main__':
+ main()
diff --git a/mk/get-win32-tarballs.sh b/mk/get-win32-tarballs.sh
deleted file mode 100755
index 4732ede52d..0000000000
--- a/mk/get-win32-tarballs.sh
+++ /dev/null
@@ -1,326 +0,0 @@
-#!/usr/bin/env bash
-
-tarball_dir='ghc-tarballs'
-missing_files=0
-pkg_variant="phyx"
-
-# see #12502
-if test -z "$FIND"; then FIND="find"; fi
-
-fail() {
- echo >&2
- echo "$1" >&2
- exit 1
-}
-
-download_file() {
- local file_url="$1"
- local dest_file="$2"
- local description="$3"
- local extra_curl_opts="$4"
- local backup_url="$5"
- local dest_dir="$(dirname $dest_file)"
-
- if ! test -f "${dest_file}"
- then
- local curl_cmd="curl -f -L ${file_url} -o ${dest_file} --create-dirs -# ${extra_curl_opts}"
- if test -n "${backup_url}"; then
- local curl_cmd_bnk="curl -f -L ${backup_url} -o ${dest_file} --create-dirs -# ${extra_curl_opts}"
- else
- local curl_cmd_bnk="true"
- fi
-
- if test "$download" = "0"
- then
- echo "ERROR: Missing ${description}" >&2
- echo "${file_url}"
- missing_files=1
- return
- else
- echo "Downloading ${description} to ${dest_dir}..."
- $curl_cmd || (echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk) || {
- rm -f "${dest_file}"
- fail "ERROR: Download failed."
- exit 1
- }
- fi
- fi
-
- local sig_file="${dest_file}.sig"
- if test "$sigs" = "1" -a ! -f "$sig_file"
- then
- echo "Downloading ${description} (signature) to ${dest_dir}..."
- local curl_cmd="curl -f -L ${file_url}.sig -o ${sig_file} --create-dirs -# ${extra_curl_opts}"
- if test -n "${backup_url}"; then
- local curl_cmd_bnk="curl -f -L "${backup_url}.sig" -o ${sig_file} --create-dirs -# ${extra_curl_opts}"
- else
- local curl_cmd_bnk="true"
- fi
- $curl_cmd || (echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk) || {
- rm -f "${dest_file}.sig"
- fail "ERROR: Download failed."
- exit 1
- }
- fi
-
- if test "$verify" = "1"
- then
- grep "${dest_file}$" mk/win32-tarballs.md5sum | md5sum --quiet -c - ||
- fail "ERROR: ${description} appears to be corrupted, please delete it and try again."
- fi
-}
-
-download_mingw() {
- local mingw_base_url_primary="https://downloads.haskell.org/~ghc/mingw"
- local mingw_base_url_secondary="http://repo.msys2.org/mingw"
-
- if test "$mingw_arch" = "sources"
- then
- mingw_url_tmp=`echo "$1" | sed -e 's/-any\.pkg\.tar\.xz/\.src\.tar\.gz/' \
- -e 's/-sources-/-/' \
- -e 's/-libwinpthread-git-/-winpthreads-git-/' `
- local mingw_url="${mingw_base_url_primary}/${mingw_url_tmp}"
- local mingw_url_backup="${mingw_base_url_secondary}/${mingw_url_tmp}"
- else
- local mingw_url="${mingw_base_url_primary}/$1"
- local mingw_url_backup="${mingw_base_url_secondary}/$1"
- fi
-
- local mingw_toolchain="$(basename $mingw_url)"
- local mingw_w64="${tarball_dir}/${tarball_dest_dir}/${mingw_toolchain}"
-
- download_file "${mingw_url}" "${mingw_w64}" "${mingw_toolchain}" "" "${mingw_url_backup}"
-
- # Mark the tree as needing updates by deleting the folder
- if test -d inplace/mingw && test inplace/mingw -ot "$mingw_w64" ; then
- echo "In-tree MinGW-w64 tree requires updates..."
- rm -rf inplace/mingw
- fi
-}
-
-download_tarballs() {
- local package_prefix="mingw-w64"
- local format_url="/${mingw_arch}/${package_prefix}-${mingw_arch}"
-
- download_mingw "${format_url}-crt-git-7.0.0.5491.fe45801e-1-any.pkg.tar.xz"
- download_mingw "${format_url}-winpthreads-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz"
- download_mingw "${format_url}-headers-git-7.0.0.5490.9ec54ed1-1-any.pkg.tar.xz"
- download_mingw "${format_url}-libwinpthread-git-7.0.0.5480.e14d23be-1-any.pkg.tar.xz"
- download_mingw "${format_url}-zlib-1.2.8-9-any.pkg.tar.xz"
- download_mingw "${format_url}-isl-0.21-1-any.pkg.tar.xz"
- download_mingw "${format_url}-mpfr-4.0.2-2-any.pkg.tar.xz"
- download_mingw "${format_url}-gmp-6.1.2-1-any.pkg.tar.xz"
- download_mingw "${format_url}-binutils-2.32-3-$pkg_variant.pkg.tar.xz"
- download_mingw "${format_url}-libidn2-2.2.0-1-any.pkg.tar.xz"
- download_mingw "${format_url}-gcc-9.2.0-1-$pkg_variant.pkg.tar.xz"
- download_mingw "${format_url}-mpc-1.1.0-1-any.pkg.tar.xz"
- download_mingw "${format_url}-windows-default-manifest-6.4-3-any.pkg.tar.xz"
-
- # Upstream is unfortunately quite inconsistent in naming
- if test "$mingw_arch" != "sources"; then
- download_mingw "${format_url}-gcc-libs-9.2.0-1-$pkg_variant.pkg.tar.xz"
- fi
-
- if ! test "$missing_files" = "0"
- then
- exit 2
- fi
-}
-
-download_i386() {
- mingw_arch="i686"
- tarball_dest_dir="mingw-w64/i686"
- download_tarballs
-}
-
-download_x86_64() {
- mingw_arch="x86_64"
- tarball_dest_dir="mingw-w64/x86_64"
- download_tarballs
-}
-
-download_sources() {
- mingw_arch="sources"
- tarball_dest_dir="mingw-w64/sources"
- download_tarballs
-}
-
-sync_binaries_and_sources() {
- gpg --recv-key 5F92EFC1A47D45A1
-
- # ensure sources are downloaded
- sigs=1
- download_i386
- download_x86_64
- verify=0
- download_sources
-
- for f in $($FIND ghc-tarballs/mingw-w64 -iname '*.sig'); do
- echo "Verifying $f"
- gpg --verify $f
- done
-
- md5sum `$FIND ghc-tarballs -type f -a -not -iname '*.sig'` >| mk/win32-tarballs.md5sum
- chmod -R ugo+rX ghc-tarballs
-
- rsync -av ghc-tarballs/mingw-w64/* downloads.haskell.org:public_html/mingw
- for f in $($FIND ghc-tarballs/mingw-w64); do
- curl -XPURGE http://downloads.haskell.org/~ghc/mingw/$f
- done
-}
-
-patch_single_file () {
- local patcher_base="$1"
- local filename=$(readlink -f "$2")
- local filepath=$(dirname "$filename")
- local patcher="$patcher_base/iat-patcher.exe"
- $patcher install "$filename" > /dev/null
- rm -f "$filename.bak"
- for file in $patcher_base/*.dll; do cp -f "$file" "${filepath}"; done
- echo "Patched '$filename'"
-}
-
-patch_tarball () {
- local tarball_name="$1"
- local filename=$(basename "$tarball_name")
- local filepath=$(dirname "$tarball_name")
- local newfile=`echo "$filepath/$filename" | sed -e 's/-any/-phyx/'`
- local arch=""
-
- echo "=> ${filename}"
-
- case $1 in
- *x86_64*)
- arch="x86_64"
- ;;
- *i686*)
- arch="i686"
- ;;
- *)
- echo "unknown architecture detected. Stopping."
- exit 1
- ;;
- esac
-
- local base="$(pwd)"
- local patcher_base="$(pwd)/ghc-tarballs/ghc-jailbreak/$arch"
- local tmpdir="ghc-tarballs/tmpdir"
- mkdir -p $tmpdir
- cd $tmpdir
- tar xJf "$base/$tarball_name"
- find . -iname "*.exe" -exec bash -c \
- 'patch_single_file "'"${patcher_base}"'" "$0"' {} \;
- tar cJf "$base/$newfile" .
- cd "$base"
- rm -rf $tmpdir
- gpg --output "$base/${newfile}.sig" --detach-sig "$base/$newfile"
- rm -f "$base/$tarball_name"
-}
-
-show_hashes_for_binaries() {
- $FIND ghc-tarballs/ -iname "*.*" | xargs md5sum | grep -v "\.sig" | sed -s "s/\*//"
-}
-
-usage() {
- echo "$0 - Download GHC mingw toolchain tarballs"
- echo
- echo "Usage: $0 <action> [<arch>]"
- echo
- echo "Where <action> is one of,"
- echo ""
- echo " download download the necessary tarballs for the given architecture"
- echo " fetch download the necessary tarballs for the given architecture but doesn't verify their md5."
- echo " grab download the necessary tarballs using patched toolchains for the given architecture but doesn't verify their md5."
- echo " verify verify the existence and correctness of the necessary tarballs"
- echo " patch jailbreak the binaries in the tarballs and remove MAX_PATH limitations."
- echo " hash generate md5 hashes for inclusion in win32-tarballs.md5sum"
- echo " sync upload packages downloaded with 'fetch mirror' to haskell.org"
- echo ""
- echo "and <arch> is one of i386, x86_64,all or mirror (which includes sources)"
-}
-
-case $1 in
- download)
- download=1
- verify=1
- sigs=0
- ;;
- fetch)
- download=1
- verify=
- ;;
- grab)
- download=1
- verify=0
- pkg_variant="any"
- ;;
- verify)
- download=0
- verify=1
- ;;
- sync)
- download=1
- verify=0
- sync=1
- ;;
- hash)
- show_hashes_for_binaries
- exit 1
- ;;
- # This routine will download the latest ghc-jailbreak and unpack binutils and
- # the ghc tarballs and patches every .exe in each. Along with this is copies
- # two dlls in every folder that it patches a .exe in. Afterwards it re-creates
- # the tarballs and generates a new signature file.
- patch)
- export -f patch_tarball
- export -f patch_single_file
-
- echo "Downloading ghc-jailbreak..."
- curl -f -L https://mistuke.blob.core.windows.net/binaries/ghc-jailbreak-0.3.tar.gz \
- -o ghc-tarballs/ghc-jailbreak/ghc-jailbreak.tar.gz --create-dirs -#
- tar -C ghc-tarballs/ghc-jailbreak/ -xf ghc-tarballs/ghc-jailbreak/ghc-jailbreak.tar.gz
-
- find ghc-tarballs/mingw-w64/ \( -iname "*binutils*.tar.xz" \
- -o -iname "*gcc*.tar.xz" \) \
- -exec bash -c 'patch_tarball "$0"' {} \;
-
- rm -rf ghc-tarballs/ghc-jailbreak
-
- echo "Finished tarball generation, toolchain has been pre-patched."
- exit 0
- ;;
- *)
- usage
- exit 1
- ;;
-esac
-
-case $2 in
- i386)
- download_i386
- ;;
- x86_64)
- download_x86_64
- ;;
- all)
- download_i386
- download_x86_64
- ;;
- mirror)
- sigs=1
- download_i386
- download_x86_64
- verify=0
- sigs=0
- download_sources
- show_hashes_for_binaries
- ;;
- *)
- if test "$sync" = "1"; then
- sync_binaries_and_sources
- else
- usage
- exit 1
- fi
- ;;
-esac