diff options
| -rw-r--r-- | .gitlab-ci.yml | 296 | ||||
| -rwxr-xr-x | .gitlab/ci.sh | 428 | ||||
| -rw-r--r-- | .gitlab/darwin-init.sh | 41 | ||||
| -rwxr-xr-x | .gitlab/prepare-system.sh | 99 | ||||
| -rw-r--r-- | .gitlab/win32-init.sh | 47 |
5 files changed, 575 insertions, 336 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 292ff7db96..990fa11f91 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,17 +5,16 @@ variables: DOCKER_REV: 408eff66aef6ca2b44446c694c5a56d6ca0460cc # Sequential version number capturing the versions of all tools fetched by - # .gitlab/win32-init.sh. + # .gitlab/ci.sh. WINDOWS_TOOLCHAIN_VERSION: 1 # Disable shallow clones; they break our linting rules GIT_DEPTH: 0 -before_script: - - git submodule sync --recursive - - git submodule update --init --recursive - - git checkout .gitmodules - - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" + # Overridden by individual jobs + CONFIGURE_ARGS: "" + + GIT_SUBMODULE_STRATEGY: "recursive" stages: - lint # Source linting @@ -37,6 +36,9 @@ stages: - web .release: &release + variables: + BUILD_FLAVOUR: "perf" + FLAVOUR: "perf" artifacts: when: always expire_in: 1 year @@ -194,26 +196,10 @@ lint-release-changelogs: variables: FLAVOUR: "validate" script: - - cabal update - - git clean -xdf && git submodule foreach git clean -xdf - - .gitlab/prepare-system.sh - - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi - - ./boot - - ./configure $CONFIGURE_ARGS - - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist - - mv _build/bindist/ghc*.tar.xz ghc.tar.xz - - export TOP=$(pwd) - - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../ - - | - # Prepare to push git notes. - export METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv - git config user.email "ben+ghc-ci@smart-cactus.org" - git config user.name "GHC GitLab CI" - .gitlab/test-metrics.sh pull - - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc || (.gitlab/test-metrics.sh push && false) - - | - # Push git notes. - .gitlab/test-metrics.sh push + - .gitlab/ci.sh setup + - .gitlab/ci.sh configure + - .gitlab/ci.sh build_hadrian + - .gitlab/ci.sh test_hadrian cache: key: hadrian paths: @@ -238,6 +224,8 @@ lint-release-changelogs: - git submodule update --init --recursive - git checkout .gitmodules - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" + after_script: + - .gitlab/ci.sh clean tags: - x86_64-linux @@ -270,7 +258,7 @@ hadrian-ghc-in-ghci: - cabal update - cd hadrian; cabal new-build --project-file=ci.project; cd .. - git clean -xdf && git submodule foreach git clean -xdf - - .gitlab/prepare-system.sh + - .gitlab/ci.sh setup - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi - ./boot - ./configure $CONFIGURE_ARGS @@ -289,28 +277,12 @@ hadrian-ghc-in-ghci: <<: *only-default variables: TEST_TYPE: test - before_script: - - git clean -xdf && git submodule foreach git clean -xdf + MAKE_ARGS: "-Werror" script: - - ./boot - - ./configure $CONFIGURE_ARGS - - | - THREADS=`mk/detect-cpu-count.sh` - make V=0 -j$THREADS WERROR=-Werror - - make binary-dist-prep TAR_COMP_OPTS="-1" - - make test_bindist TEST_PREP=YES - - | - # Prepare to push git notes. - METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv - git config user.email "ben+ghc-ci@smart-cactus.org" - git config user.name "GHC GitLab CI" - .gitlab/test-metrics.sh pull - - | - THREADS=`mk/detect-cpu-count.sh` - make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE || (METRICS_FILE=$METRICS_FILE .gitlab/test-metrics.sh push && false) - - | - # Push git notes. - METRICS_FILE=$METRICS_FILE .gitlab/test-metrics.sh push + - .gitlab/ci.sh setup + - .gitlab/ci.sh configure + - .gitlab/ci.sh build_make + - .gitlab/ci.sh test_make dependencies: [] artifacts: reports: @@ -322,6 +294,79 @@ hadrian-ghc-in-ghci: - performance-metrics.tsv ################################# +# x86_64-freebsd +################################# + +.build-x86_64-freebsd: + extends: .validate + tags: + - x86_64-freebsd + allow_failure: true + variables: + # N.B. we use iconv from ports as I see linker errors when we attempt + # to use the "native" iconv embedded in libc as suggested by the + # porting guide [1]. + # [1] https://www.freebsd.org/doc/en/books/porters-handbook/using-iconv.html) + CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib" + GHC_VERSION: 8.6.3 + CABAL_INSTALL_VERSION: 3.0.0.0 + BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz" + TEST_ENV: "x86_64-freebsd" + BUILD_FLAVOUR: "validate" + after_script: + - cp -Rf $HOME/.cabal cabal-cache + - .gitlab/ci.sh clean + artifacts: + when: always + expire_in: 2 week + cache: + key: "freebsd-$GHC_VERSION" + paths: + - cabal-cache + - toolchain + +# Disabled due to lack of builder capacity +.validate-x86_64-freebsd: + extends: .build-x86_64-freebsd + stage: full-build + +nightly-x86_64-freebsd: + <<: *nightly + extends: .build-x86_64-freebsd + stage: full-build + +.build-x86_64-freebsd-hadrian: + extends: .validate-hadrian + stage: full-build + tags: + - x86_64-freebsd + allow_failure: true + variables: + CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib" + HADRIAN_ARGS: "--docs=no-sphinx" + GHC_VERSION: 8.6.3 + CABAL_INSTALL_VERSION: 3.0.0.0 + BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz" + TEST_ENV: "x86_64-freebsd-hadrian" + FLAVOUR: "validate" + after_script: + - cp -Rf $HOME/.cabal cabal-cache + - .gitlab/ci.sh clean + artifacts: + when: always + expire_in: 2 week + cache: + key: "freebsd-$GHC_VERSION" + paths: + - cabal-cache + - toolchain + +# Disabled due to lack of builder capacity +.validate-x86_64-freebsd-hadrian: + extends: .build-x86_64-freebsd-hadrian + stage: full-build + +################################# # x86_64-darwin ################################# @@ -332,25 +377,18 @@ validate-x86_64-darwin: - x86_64-darwin variables: GHC_VERSION: 8.6.5 - CABAL_INSTALL_VERSION: 2.4.1.0 + CABAL_INSTALL_VERSION: 3.0.0.0 BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-apple-darwin.tar.xz" MACOSX_DEPLOYMENT_TARGET: "10.7" # Only Sierra and onwards supports clock_gettime. See #12858 ac_cv_func_clock_gettime: "no" LANG: "en_US.UTF-8" - CONFIGURE_ARGS: --with-intree-gmp + CONFIGURE_ARGS: "--with-intree-gmp" TEST_ENV: "x86_64-darwin" - before_script: - - git clean -xdf && git submodule foreach git clean -xdf - - git submodule sync --recursive - - git submodule update --init --recursive - - git checkout .gitmodules - - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" - - - bash .gitlab/darwin-init.sh - - PATH="`pwd`/toolchain/bin:$PATH" + BUILD_FLAVOUR: "perf" after_script: - cp -Rf $HOME/.cabal cabal-cache + - .gitlab/ci.sh clean artifacts: when: always expire_in: 2 week @@ -374,26 +412,14 @@ validate-x86_64-darwin: CONFIGURE_ARGS: --with-intree-gmp TEST_ENV: "x86_64-darwin-hadrian" FLAVOUR: "validate" - before_script: - - git clean -xdf && git submodule foreach git clean -xdf - - git submodule sync --recursive - - git submodule update --init --recursive - - git checkout .gitmodules - - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" - - - bash .gitlab/darwin-init.sh - - PATH="`pwd`/toolchain/bin:$PATH" script: - - cabal update - - ./boot - - ./configure $CONFIGURE_ARGS - - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist - - mv _build/bindist/ghc*.tar.xz ghc.tar.xz - - export TOP=$(pwd) - - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../ - - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc + - .gitlab/ci.sh setup + - .gitlab/ci.sh configure + - .gitlab/ci.sh build_hadrian + - .gitlab/ci.sh test_hadrian after_script: - cp -Rf $HOME/.cabal cabal-cache + - .gitlab/ci.sh clean artifacts: when: always expire_in: 2 week @@ -407,19 +433,15 @@ validate-x86_64-darwin: extends: .validate tags: - x86_64-linux + variables: + BUILD_FLAVOUR: "perf" before_script: - - git clean -xdf && git submodule foreach git clean -xdf - - git submodule sync --recursive - - git submodule update --init --recursive - - git checkout .gitmodules - - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" # Build hyperlinked sources for documentation when building releases - | if [[ -n "$CI_COMMIT_TAG" ]]; then - echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk + HADDOCK_HYPERLINKED_SOURCES=1 fi - - .gitlab/prepare-system.sh # workaround for docker permissions - sudo chown ghc:ghc -R . after_script: @@ -771,57 +793,44 @@ validate-x86_64-linux-fedora27: before_script: - git clean -xdf - # Use a local temporary directory to ensure that concurrent builds don't - # interfere with one another - - | - mkdir tmp - set TMP=%cd%\tmp - set TEMP=%cd%\tmp - - - set PATH=C:\msys64\usr\bin;%PATH% - # On Windows submodules can inexplicably get into funky states where git - # believes that the submodule is initialized yet its associated repository - # is not valid. Avoid failing in this case with the following insanity. - - | - bash -c 'git submodule sync --recursive || git submodule deinit --force --all' - bash -c 'git submodule update --init --recursive' - git submodule foreach git clean -xdf - - git checkout .gitmodules - - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" - - bash .gitlab/win32-init.sh + # Setup toolchain + - bash .gitlab/ci.sh setup after_script: - - rd /s /q tmp - - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache - - bash -c 'make clean || true' + - | + Copy-Item -Recurse -Path $Env:APPDATA\cabal -Destination cabal-cache + - bash .gitlab/ci.sh clean dependencies: [] variables: #FORCE_SYMLINKS: 1 LANG: "en_US.UTF-8" SPHINXBUILD: "/mingw64/bin/sphinx-build.exe" + CABAL_INSTALL_VERSION: 3.0.0.0 + GHC_VERSION: "8.6.5" cache: paths: - cabal-cache - - ghc-8.6.5 + - toolchain - ghc-tarballs .build-windows-hadrian: extends: .build-windows stage: full-build variables: - GHC_VERSION: "8.6.5" FLAVOUR: "validate" + # skipping perf tests for now since we build a quick-flavoured GHC, + # which might result in some broken perf tests? + HADRIAN_ARGS: "--docs=no-sphinx --skip-perf" + # due to #16574 this currently fails allow_failure: true + script: - - | - python boot - bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex' - - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist" - - mv _build/bindist/ghc*.tar.xz ghc.tar.xz - - bash -c "export TOP=$(pwd); cd _build/bindist/ghc-*/ && PATH=$TOP/toolchain/bin:$PATH ./configure --prefix=$TOP/_build/install && make install && cd ../../../" - - bash -c "export TOP=$(pwd); PATH=$TOP/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc" + - bash .gitlab/ci.sh configure + - bash .gitlab/ci.sh build_hadrian + - bash .gitlab/ci.sh test_hadrian tags: - - x86_64-windows + - new-x86_64-windows + - test artifacts: reports: junit: junit.xml @@ -833,7 +842,6 @@ validate-x86_64-linux-fedora27: validate-x86_64-windows-hadrian: extends: .build-windows-hadrian - stage: full-build variables: MSYSTEM: MINGW64 TEST_ENV: "x86_64-windows-hadrian" @@ -856,20 +864,14 @@ nightly-i386-windows-hadrian: stage: full-build variables: BUILD_FLAVOUR: "quick" - GHC_VERSION: "8.6.5" BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-mingw32.tar.xz" script: - - | - python boot - bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex $CONFIGURE_ARGS' - - bash -c "echo include mk/flavours/${BUILD_FLAVOUR}.mk > mk/build.mk" - - bash -c "echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk" - - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`" - - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist-prep TAR_COMP_OPTS=-1" - - bash -c "PATH=`pwd`/toolchain/bin:$PATH make test_bindist TEST_PREP=YES" - - bash -c 'make V=0 test PYTHON=/mingw64/bin/python3 THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml' + - bash .gitlab/ci.sh configure + - bash .gitlab/ci.sh build_make + - bash .gitlab/ci.sh test_make tags: - - x86_64-windows + - new-x86_64-windows + - test artifacts: when: always expire_in: 2 week @@ -881,64 +883,60 @@ nightly-i386-windows-hadrian: - "ghc-x86_64-mingw32.tar.xz" - junit.xml -validate-x86_64-windows: +.build-x86_64-windows-make: extends: .build-windows-make variables: MSYSTEM: MINGW64 - CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32" TEST_ENV: "x86_64-windows" cache: key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION" +validate-x86_64-windows: + extends: .build-x86_64-windows-make + nightly-x86_64-windows: - extends: .build-windows-make + extends: .build-x86_64-windows-make stage: full-build variables: BUILD_FLAVOUR: "validate" - MSYSTEM: MINGW64 - CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32" only: variables: - $NIGHTLY - cache: - key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION" # Normal Windows validate builds are profiled; that won't do for releases. release-x86_64-windows: <<: *release extends: validate-x86_64-windows variables: - MSYSTEM: MINGW64 BUILD_FLAVOUR: "perf" - CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32" - TEST_ENV: "x86_64-windows" -release-i386-windows: - <<: *release + +.build-i386-windows-make: extends: .build-windows-make variables: MSYSTEM: MINGW32 - BUILD_FLAVOUR: "perf" - CONFIGURE_ARGS: "--target=i386-unknown-mingw32" # Due to #15934 BUILD_PROF_LIBS: "NO" TEST_ENV: "i386-windows" cache: key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION" +validate-i386-windows: + extends: .build-i386-windows-make + variables: + BUILD_FLAVOUR: "perf" + +release-i386-windows: + <<: *release + extends: .build-i386-windows-make + variables: + BUILD_FLAVOUR: "perf" + nightly-i386-windows: - extends: .build-windows-make + extends: .build-i386-windows-make only: variables: - $NIGHTLY - variables: - MSYSTEM: MINGW32 - CONFIGURE_ARGS: "--target=i386-unknown-mingw32" - # Due to #15934 - BUILD_PROF_LIBS: "NO" - TEST_ENV: "i386-windows" - cache: - key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION" ############################################################ # Cleanup diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh new file mode 100755 index 0000000000..90d789f040 --- /dev/null +++ b/.gitlab/ci.sh @@ -0,0 +1,428 @@ +#!/usr/bin/env bash + +# This is the primary driver of the GitLab CI infrastructure. + +set -e -o pipefail + +# Configuration: +hackage_index_state="@1579718451" + +# Colors +BLACK="0;30" +GRAY="1;30" +RED="0;31" +LT_RED="1;31" +BROWN="0;33" +LT_BROWN="1;33" +GREEN="0;32" +LT_GREEN="1;32" +BLUE="0;34" +LT_BLUE="1;34" +PURPLE="0;35" +LT_PURPLE="1;35" +CYAN="0;36" +LT_CYAN="1;36" +WHITE="1;37" +LT_GRAY="0;37" + +# GitLab Pipelines log section delimiters +# https://gitlab.com/gitlab-org/gitlab-foss/issues/14664 +start_section() { + name="$1" + echo -e "section_start:$(date +%s):$name\015\033[0K" +} + +end_section() { + name="$1" + echo -e "section_end:$(date +%s):$name\015\033[0K" +} + +echo_color() { + local color="$1" + local msg="$2" + echo -e "\e[${color}m${msg}\e[0m" +} + +error() { echo_color "${RED}" "$1"; } +warn() { echo_color "${LT_BROWN}" "$1"; } +info() { echo_color "${LT_BLUE}" "$1"; } + +fail() { error "error: $1"; exit 1; } + +function run() { + info "Running $*..." + "$@" || ( error "$* failed"; return 1; ) +} + +if [ -z "$GHC_VERSION" ]; then + fail "GHC_VERSION not set" +fi + +TOP="$(pwd)" + +function mingw_init() { + case "$MSYSTEM" in + MINGW32) + triple="i386-unknown-mingw32" + boot_triple="i386-unknown-mingw32" # triple of bootstrap GHC + ;; + MINGW64) + triple="x86_64-unknown-mingw32" + boot_triple="x86_64-unknown-mingw32" # triple of bootstrap GHC + ;; + *) + fail "win32-init: Unknown MSYSTEM $MSYSTEM" + ;; + esac + + # Bring mingw toolchain into PATH. + # This is extracted from /etc/profile since this script inexplicably fails to + # run under gitlab-runner. + source /etc/msystem + MINGW_MOUNT_POINT="${MINGW_PREFIX}" + PATH="$MINGW_MOUNT_POINT/bin:$PATH" +} + +# This will contain GHC's local native toolchain +toolchain="$TOP/toolchain" +mkdir -p $toolchain/bin +PATH="$toolchain/bin:$PATH" + +export METRICS_FILE="$CI_PROJECT_DIR/performance-metrics.tsv" + +# Use a local temporary directory to ensure that concurrent builds don't +# interfere with one another +mkdir -p $TOP/tmp +export TMP="$TOP/tmp" +export TEMP="$TOP/tmp" + +function darwin_setup() { + # It looks like we already have python2 here and just installing python3 + # does not work. + brew upgrade python + brew install ghc cabal-install ncurses gmp + + pip3 install sphinx + # PDF documentation disabled as MacTeX apparently doesn't include xelatex. + #brew cask install mactex +} + +function show_tool() { + local tool="$1" + info "$tool = ${!tool}" + ${!tool} --version +} + +function set_toolchain_paths() { + needs_toolchain=1 + case "$(uname)" in + Linux) needs_toolchain="" ;; + *) ;; + esac + + if [[ -n "$needs_toolchain" ]]; then + # These are populated by setup_toolchain + export GHC="$toolchain/bin/ghc$exe" + export CABAL="$toolchain/bin/cabal$exe" + export HAPPY="$toolchain/bin/happy$exe" + export ALEX="$toolchain/bin/alex$exe" + else + export GHC="$(which ghc)" + export CABAL="/usr/local/bin/cabal" + export HAPPY="$HOME/.cabal/bin/happy" + export ALEX="$HOME/.cabal/bin/alex" + fi + + # FIXME: Temporarily use ghc from ports + case "$(uname)" in + FreeBSD) GHC="/usr/local/bin/ghc" ;; + *) ;; + esac +} + +# Extract GHC toolchain +function setup() { + if [ -d "$TOP/cabal-cache" ]; then + info "Extracting cabal cache..." + cp -Rf cabal-cache $cabal_dir + fi + + if [[ -n "$needs_toolchain" ]]; then + setup_toolchain + fi + case "$(uname)" in + Darwin) darwin_setup ;; + *) ;; + esac + + # Make sure that git works + git config user.email "ghc-ci@gitlab-haskell.org" + git config user.name "GHC GitLab CI" + + info "=====================================================" + info "Toolchain versions" + info "=====================================================" + show_tool GHC + show_tool CABAL + show_tool HAPPY + show_tool ALEX +} + +function fetch_ghc() { + local v="$GHC_VERSION" + if [[ -z "$v" ]]; then + fail "GHC_VERSION is not set" + fi + + if [ ! -e "$GHC" ]; then + start_section "fetch GHC" + url="https://downloads.haskell.org/~ghc/${GHC_VERSION}/ghc-${GHC_VERSION}-${boot_triple}.tar.xz" + info "Fetching GHC binary distribution from $url..." + curl $url > ghc.tar.xz || fail "failed to fetch GHC binary distribution" + tar -xJf ghc.tar.xz || fail "failed to extract GHC binary distribution" + case "$(uname)" in + MSYS_*|MINGW*) + cp -r ghc-${GHC_VERSION}/* $toolchain + ;; + *) + pushd ghc-${GHC_VERSION} + ./configure --prefix=$toolchain + $MAKE install + popd + ;; + esac + rm -Rf ghc-${GHC_VERSION} ghc.tar.xz + end_section "fetch GHC" + fi + +} + +function fetch_cabal() { + local v="$CABAL_INSTALL_VERSION" + if [[ -z "$v" ]]; then + fail "CABAL_INSTALL_VERSION is not set" + fi + + if [ ! -e "$CABAL" ]; then + start_section "fetch GHC" + case "$(uname)" in + # N.B. Windows uses zip whereas all others use .tar.xz + MSYS_*|MINGW*) + case "$MSYSTEM" in + MINGW32) cabal_arch="i386" ;; + MINGW64) cabal_arch="x86_64" ;; + *) fail "unknown MSYSTEM $MSYSTEM" ;; + esac + url="https://downloads.haskell.org/~cabal/cabal-install-$v/cabal-install-$v-$cabal_arch-unknown-mingw32.zip" + info "Fetching cabal binary distribution from $url..." + curl $url > $TMP/cabal.zip + unzip $TMP/cabal.zip + mv cabal.exe $CABAL + ;; + *) + local base_url="https://downloads.haskell.org/~cabal/cabal-install-$v/" + case "$(uname)" in + Darwin) cabal_url="$base_url/cabal-install-$v-x86_64-apple-darwin17.7.0.tar.xz" ;; + FreeBSD) + #cabal_url="$base_url/cabal-install-$v-x86_64-portbld-freebsd.tar.xz" ;; + cabal_url="http://home.smart-cactus.org/~ben/ghc/cabal-install-3.0.0.0-x86_64-portbld-freebsd.tar.xz" ;; + *) fail "don't know where to fetch cabal-install for $(uname)" + esac + echo "Fetching cabal-install from $cabal_url" + curl $cabal_url > cabal.tar.xz + tar -xJf cabal.tar.xz + mv cabal $toolchain/bin + ;; + esac + end_section "fetch GHC" + fi +} + +# For non-Docker platforms we prepare the bootstrap toolchain +# here. For Docker platforms this is done in the Docker image +# build. +function setup_toolchain() { + fetch_ghc + fetch_cabal + cabal_install="$CABAL v2-install --index-state=$hackage_index_state --installdir=$toolchain/bin" + # Avoid symlinks on Windows + case "$(uname)" in + MSYS_*|MINGW*) cabal_install="$cabal_install --install-method=copy" ;; + *) ;; + esac + + if [ ! -e "$HAPPY" ]; then + info "Building happy..." + cabal update + $cabal_install happy + fi + + if [ ! -e "$ALEX" ]; then + info "Building alex..." + cabal update + $cabal_install alex + fi +} + +function cleanup_submodules() { + start_section "clean submodules" + info "Cleaning submodules..." + # On Windows submodules can inexplicably get into funky states where git + # believes that the submodule is initialized yet its associated repository + # is not valid. Avoid failing in this case with the following insanity. + git submodule sync --recursive || git submodule deinit --force --all + git submodule update --init --recursive + git submodule foreach git clean -xdf + end_section "clean submodules" +} + +function prepare_build_mk() { + if [[ -z "$BUILD_FLAVOUR" ]]; then fail "BUILD_FLAVOUR is not set"; fi + if [[ -z ${BUILD_SPHINX_HTML:-} ]]; then BUILD_SPHINX_HTML=YES; fi + if [[ -z ${BUILD_SPHINX_PDF:-} ]]; then BUILD_SPHINX_PDF=YES; fi + if [[ -z ${INTEGER_LIBRARY:-} ]]; then INTEGER_LIBRARY=integer-gmp; fi + + cat > mk/build.mk <<EOF +V=1 +HADDOCK_DOCS=YES +LATEX_DOCS=YES +HSCOLOUR_SRCS=YES +BUILD_SPHINX_HTML=$BUILD_SPHINX_HTML +BUILD_SPHINX_PDF=$BUILD_SPHINX_PDF +BeConservative=YES +INTEGER_LIBRARY=$INTEGER_LIBRARY +XZ_CMD=$XZ + +BuildFlavour=$BUILD_FLAVOUR +ifneq "\$(BuildFlavour)" "" +include mk/flavours/\$(BuildFlavour).mk +endif +GhcLibHcOpts+=-haddock +EOF + + if [ -n "$HADDOCK_HYPERLINKED_SOURCES" ]; then + echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk + fi + + case "$(uname)" in + Darwin) echo "libraries/integer-gmp_CONFIGURE_OPTS += --configure-option=--with-intree-gmp" >> mk/build.mk ;; + *) ;; + esac + + info "build.mk is:" + cat mk/build.mk +} + +function configure() { + start_section "booting" + run python3 boot + end_section "booting" + + start_section "configuring" + run ./configure \ + --enable-tarballs-autodownload \ + --target=$triple \ + $CONFIGURE_ARGS \ + GHC=$GHC \ + HAPPY=$HAPPY \ + ALEX=$ALEX \ + || ( cat config.log; fail "configure failed" ) + end_section "configuring" +} + +function build_make() { + prepare_build_mk + if [[ -z "$BIN_DIST_PREP_TAR_COMP" ]]; then + fail "BIN_DIST_PREP_TAR_COMP is not set" + fi + + echo "include mk/flavours/${BUILD_FLAVOUR}.mk" > mk/build.mk + echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk + run $MAKE -j$(mk/detect-cpu-count.sh) $MAKE_ARGS + run $MAKE -j$(mk/detect-cpu-count.sh) binary-dist-prep TAR_COMP_OPTS=-1 + ls -lh $BIN_DIST_PREP_TAR_COMP +} + +function fetch_perf_notes() { + info "Fetching perf notes..." + $TOP/.gitlab/test-metrics.sh pull +} + +function push_perf_notes() { + info "Pushing perf notes..." + $TOP/.gitlab/test-metrics.sh push +} + +function test_make() { + run $MAKE test_bindist TEST_PREP=YES + run $MAKE V=0 test \ + THREADS=$(mk/detect-cpu-count.sh) \ + JUNIT_FILE=../../junit.xml +} + +function build_hadrian() { + if [ -z "$FLAVOUR" ]; then + fail "FLAVOUR not set" + fi + + run hadrian/build.cabal.sh \ + --flavour=$FLAVOUR \ + -j$(mk/detect-cpu-count.sh) \ + $HADRIAN_ARGS \ + binary-dist + + mv _build/bindist/ghc*.tar.xz ghc.tar.xz +} + +function test_hadrian() { + cd _build/bindist/ghc-*/ + run ./configure --prefix=$TOP/_build/install + run $MAKE install + cd ../../../ + + run hadrian/build.cabal.sh \ + --flavour=$FLAVOUR \ + -j$(mk/detect-cpu-count.sh) \ + $HADRIAN_ARGS \ + test \ + --summary-junit=./junit.xml \ + --test-compiler=$TOP/_build/install/bin/ghc +} + +function clean() { + rm -R tmp + run $MAKE --quiet clean || true + run rm -Rf _build +} + +# Determine Cabal data directory +case "$(uname)" in + MSYS_*|MINGW*) exe=".exe"; cabal_dir="$APPDATA/cabal" ;; + *) cabal_dir="$HOME/.cabal"; exe="" ;; +esac + +# Platform-specific environment initialization +MAKE="make" +case "$(uname)" in + MSYS_*|MINGW*) mingw_init ;; + Darwin) boot_triple="x86_64-apple-darwin" ;; + FreeBSD) + boot_triple="x86_64-portbld-freebsd" + MAKE="gmake" + ;; + Linux) ;; + *) fail "uname $(uname) is not supported" ;; +esac + +set_toolchain_paths + +case $1 in + setup) setup && cleanup_submodules ;; + configure) configure ;; + build_make) build_make ;; + test_make) fetch_perf_notes; test_make; push_perf_notes ;; + build_hadrian) build_hadrian ;; + test_hadrian) fetch_perf_notes; test_hadrian; push_perf_notes ;; + clean) clean ;; + *) fail "unknown mode $1" ;; +esac diff --git a/.gitlab/darwin-init.sh b/.gitlab/darwin-init.sh deleted file mode 100644 index 3dadc04dc1..0000000000 --- a/.gitlab/darwin-init.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -toolchain=`pwd`/toolchain -PATH="$toolchain/bin:$PATH" - -if [ -d "`pwd`/cabal-cache" ]; then - cp -Rf cabal-cache $HOME/.cabal -fi - -if [ ! -e $toolchain/bin/ghc ]; then - mkdir -p tmp - cd tmp - ghc_tarball="https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-x86_64-apple-darwin.tar.xz" - echo "Fetching GHC from $ghc_tarball" - curl $ghc_tarball | tar -xJ - cd ghc-$GHC_VERSION - ./configure --prefix=$toolchain - make install - cd ../.. - rm -Rf tmp -fi - -if [ ! -e $toolchain/bin/cabal ]; then - cabal_tarball="https://downloads.haskell.org/~cabal/cabal-install-$CABAL_INSTALL_VERSION/cabal-install-$CABAL_INSTALL_VERSION-x86_64-apple-darwin-sierra.tar.xz" - echo "Fetching cabal-install from $cabal_tarball" - curl $cabal_tarball | tar -xz - mv cabal $toolchain/bin -fi - -if [ ! -e $toolchain/bin/happy ]; then - cabal update - cabal new-install happy --symlink-bindir=$toolchain/bin -fi - -if [ ! -e $toolchain/bin/alex ]; then - cabal update - cabal new-install alex --symlink-bindir=$toolchain/bin -fi - diff --git a/.gitlab/prepare-system.sh b/.gitlab/prepare-system.sh deleted file mode 100755 index 4180e6029c..0000000000 --- a/.gitlab/prepare-system.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# vim: sw=2 et -set -euo pipefail - -fail() { - echo "ERROR: $*" >&2 - exit 1 -} - -hackage_index_state="@1522046735" - -if [[ -z ${BUILD_SPHINX_HTML:-} ]]; then BUILD_SPHINX_HTML=YES; fi -if [[ -z ${BUILD_SPHINX_PDF:-} ]]; then BUILD_SPHINX_PDF=YES; fi -if [[ -z ${INTEGER_LIBRARY:-} ]]; then INTEGER_LIBRARY=integer-gmp; fi -if [[ -z ${BUILD_FLAVOUR:-} ]]; then BUILD_FLAVOUR=perf; fi - -if [[ -z ${XZ:-} ]]; then - if which pxz; then - XZ="pxz" - elif which xz; then - # Check whether --threads is supported - if echo "hello" | xz --threads=$CORES >/dev/null; then - XZ="xz --threads=$CORES" - else - XZ="xz" - fi - else - echo "error: neither pxz nor xz were found" - exit 1 - fi -fi -echo "Using $XZ for compression..." - - -cat > mk/build.mk <<EOF -V=1 -HADDOCK_DOCS=YES -LATEX_DOCS=YES -HSCOLOUR_SRCS=YES -BUILD_SPHINX_HTML=$BUILD_SPHINX_HTML -BUILD_SPHINX_PDF=$BUILD_SPHINX_PDF -BeConservative=YES -INTEGER_LIBRARY=$INTEGER_LIBRARY -XZ_CMD=$XZ -EOF - -cat <<EOF >> mk/build.mk -BuildFlavour=$BUILD_FLAVOUR -ifneq "\$(BuildFlavour)" "" -include mk/flavours/\$(BuildFlavour).mk -endif -GhcLibHcOpts+=-haddock -EOF - -case "$(uname)" in - Linux) - if [[ -n ${TARGET:-} ]]; then - if [[ $TARGET = FreeBSD ]]; then - # cross-compiling to FreeBSD - echo 'HADDOCK_DOCS = NO' >> mk/build.mk - echo 'WERROR=' >> mk/build.mk - # https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables - echo 'export PATH=/opt/ghc/bin:$PATH' >> $BASH_ENV - else - fail "TARGET=$target not supported" - fi - fi - ;; - - Darwin) - if [[ -n ${TARGET:-} ]]; then - fail "uname=$(uname) not supported for cross-compilation" - fi - # It looks like we already have python2 here and just installing python3 - # does not work. - brew upgrade python - brew install ghc cabal-install ncurses gmp - - pip3 install sphinx - # PDF documentation disabled as MacTeX apparently doesn't include xelatex. - #brew cask install mactex - - cabal update - cabal install --reinstall alex happy haddock hscolour --index-state=$hackage_index_state - # put them on the $PATH, don't fail if already installed - ln -s $HOME/.cabal/bin/alex /usr/local/bin/alex || true - ln -s $HOME/.cabal/bin/happy /usr/local/bin/happy || true - ln -s $HOME/.cabal/bin/HsColour /usr/local/bin/HsColour || true - echo "libraries/integer-gmp_CONFIGURE_OPTS += --configure-option=--with-intree-gmp" >> mk/build.mk - ;; - *) - fail "uname=$(uname) not supported" -esac - -echo "=================================================" -echo "Build.mk:" -echo "" -cat mk/build.mk -echo "=================================================" diff --git a/.gitlab/win32-init.sh b/.gitlab/win32-init.sh deleted file mode 100644 index aec73ce083..0000000000 --- a/.gitlab/win32-init.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -set -e - -toolchain=`pwd`/toolchain -PATH="$toolchain/bin:/mingw64/bin:$PATH" - -if [ -d "`pwd`/cabal-cache" ]; then - cp -Rf cabal-cache $APPDATA/cabal -fi - -if [ ! -e $toolchain/bin/ghc ]; then - case $MSYSTEM in - MINGW32) - triple="i386-unknown-mingw32" - ;; - MINGW64) - triple="x86_64-unknown-mingw32" - ;; - *) - echo "win32-init: Unknown MSYSTEM $MSYSTEM" - exit 1 - ;; - esac - curl https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-$triple.tar.xz | tar -xJ - mv ghc-$GHC_VERSION toolchain -fi - -if [ ! -e $toolchain/bin/cabal ]; then - url="https://downloads.haskell.org/~cabal/cabal-install-2.4.1.0/cabal-install-2.4.1.0-x86_64-unknown-mingw32.zip" - curl $url > /tmp/cabal.zip - unzip /tmp/cabal.zip - mv cabal.exe $toolchain/bin -fi - -if [ ! -e $toolchain/bin/happy ]; then - cabal update - cabal install happy - cp $APPDATA/cabal/bin/happy $toolchain/bin -fi - -if [ ! -e $toolchain/bin/alex ]; then - cabal update - cabal install alex - cp $APPDATA/cabal/bin/alex $toolchain/bin -fi - |