Add backup url and sync support for Win32 tarball script

This imports @bgamari's sync script into the mirror script
and adds a backup url for packages.

The idea is that the URLs won't need updating when updating
the tarballs from now on.

It will first try haskell.org,
failing that it'll try repo.msys2.org

Test Plan: try new command `mk/get-win32-tarballs.sh sync`

Reviewers: austin, bgamari

Reviewed By: bgamari

Subscribers: rwbarton, thomie, bgamari

Differential Revision: https://phabricator.haskell.org/D3488

1 files changed, 59 insertions, 13 deletions

diff --git a/mk/get-win32-tarballs.sh b/mk/get-win32-tarballs.sh
index 2fd81449ce..a21cc6224b 100755
--- a/mk/get-win32-tarballs.sh
+++ b/mk/get-win32-tarballs.sh
@@ -14,11 +14,18 @@ download_file() {
     local dest_file="$2"
     local description="$3"
     local extra_curl_opts="$4"
+    local backup_url="$5"
     local dest_dir="$(dirname $dest_file)"
 
     if ! test -f "${dest_file}"
     then
         local curl_cmd="curl -L ${file_url} -o ${dest_file} --create-dirs -# ${extra_curl_opts}"
+        if test -n "${backup_url}"; then
+            local curl_cmd_bnk="curl -L ${backup_url} -o ${dest_file} --create-dirs -# ${extra_curl_opts}"
+        else
+            local curl_cmd_bnk="echo 1"
+        fi
+
         if test "$download" = "0"
         then
             echo "ERROR: Missing ${description}" >&2
@@ -27,7 +34,7 @@ download_file() {
             return
         else
             echo "Downloading ${description} to ${dest_dir}..."
-            $curl_cmd || {
+            $curl_cmd || $curl_cmd_bnk || {
                 rm -f "${dest_file}"
                 fail "ERROR: Download failed."
             }
@@ -53,19 +60,24 @@ download_file() {
 }
 
 download_mingw() {
+    local mingw_base_url_primary="https://downloads.haskell.org/~ghc/mingw"
+    local mingw_base_url_secondary="http://repo.msys2.org/mingw"
+
     if test "$mingw_arch" = "sources"
     then
-        local mingw_url=`echo "$1" | sed -e 's/-any\.pkg\.tar\.xz/\.src\.tar\.gz/' \
-                                         -e 's/-sources-/-/' \
-                                         -e 's/-libwinpthread-git-/-winpthreads-git-/' `
+        mingw_url_tmp=`echo "$1" | sed -e 's/-any\.pkg\.tar\.xz/\.src\.tar\.gz/' \
+                                       -e 's/-sources-/-/' \
+                                       -e 's/-libwinpthread-git-/-winpthreads-git-/' `
+        local mingw_url="${mingw_base_url_primary}/${mingw_url_tmp}"
     else
-        local mingw_url="$1"
+        local mingw_url="${mingw_base_url_primary}/$1"
+        local mingw_url_backup="${mingw_base_url_secondary}/$1"
     fi
 
     local mingw_toolchain="$(basename $mingw_url)"
     local mingw_w64="${tarball_dir}/${tarball_dest_dir}/${mingw_toolchain}"
 
-    download_file "${mingw_url}" "${mingw_w64}" "${mingw_toolchain}"
+    download_file "${mingw_url}" "${mingw_w64}" "${mingw_toolchain}" "" "${mingw_url_backup}"
 
     # Mark the tree as needing updates by deleting the folder
     if test -d inplace/mingw && test inplace/mingw -ot "$mingw_w64" ; then
@@ -75,10 +87,8 @@ download_mingw() {
 }
 
 download_tarballs() {
-    #local mingw_base_url="http://repo.msys2.org/mingw"
-    local mingw_base_url="https://downloads.haskell.org/~ghc/mingw"
     local package_prefix="mingw-w64"
-    local format_url="${mingw_base_url}/${mingw_arch}/${package_prefix}-${mingw_arch}"
+    local format_url="/${mingw_arch}/${package_prefix}-${mingw_arch}"
 
     download_mingw "${format_url}-crt-git-5.0.0.4795.e3d96cb1-1-any.pkg.tar.xz"
     download_mingw "${format_url}-winpthreads-git-5.0.0.4761.02bea78-1-any.pkg.tar.xz"
@@ -128,15 +138,42 @@ download_sources() {
     download_tarballs
 }
 
+sync_binaries_and_sources() {
+    gpg --recv-key  5F92EFC1A47D45A1
+
+    # ensure sources are downloaded
+    sigs=1
+    download_i386
+    download_x86_64
+    verify=0
+    download_sources
+
+    for f in $(find ghc-tarballs/mingw-w64 -iname '*.sig'); do
+        echo "Verifying $f"
+        gpg --verify $f
+    done
+
+    md5sum `find ghc-tarballs -type f -a -not -iname '*.sig'` >| mk/win32-tarballs.md5sum
+    chmod -R ugo+rX ghc-tarballs
+
+    rsync -av ghc-tarballs/mingw-w64/* downloads.haskell.org:public_html/mingw
+    for f in $(find ghc-tarballs/mingw-w64); do
+        curl -XPURGE http://downloads.haskell.org/~ghc/mingw/$f
+    done
+}
+
 usage() {
     echo "$0 - Download GHC mingw toolchain tarballs"
     echo
-    echo "Usage: $0 <action> <arch>"
+    echo "Usage: $0 <action> [<arch>]"
     echo
     echo "Where <action> is one of,"
+    echo ""
     echo "    download     download the necessary tarballs for the given architecture"
-    echo "    fetch        download the necessary tarballs for the given architecture but doesn't verify their md5."d
+    echo "    fetch        download the necessary tarballs for the given architecture but doesn't verify their md5."
     echo "    verify       verify the existence and correctness of the necessary tarballs"
+    echo "    sync         upload packages downloaded with 'fetch mirror' to haskell.org"
+    echo ""
     echo "and <arch> is one of i386, x86_64,all or mirror (which includes sources)"
 }
 
@@ -154,6 +191,11 @@ case $1 in
         download=0
         verify=1
         ;;
+    sync)
+        download=1
+        verify=0
+        sync=1
+        ;;
     *)
         usage
         exit 1
@@ -179,7 +221,11 @@ case $2 in
         download_sources
         ;;
     *)
-        usage
-        exit 1
+        if test "$sync" = "1"; then
+            sync_binaries_and_sources
+        else
+            usage
+            exit 1
+        fi
         ;;
 esac