diff options
author | Ben Gamari <ben@smart-cactus.org> | 2019-04-04 11:14:03 -0400 |
---|---|---|
committer | Ben Gamari <ben@smart-cactus.org> | 2019-04-17 10:57:23 -0400 |
commit | 7ccdb7de2d03ca4d4878a14abcc655c088181534 (patch) | |
tree | 98e0617877ec6a27f182c04e2bc82decd8c93931 | |
parent | 4e7340595584b45c49f62ee270aa68de782e07aa (diff) | |
download | haskell-7ccdb7de2d03ca4d4878a14abcc655c088181534.tar.gz |
gitlab-ci: Backport from master
-rw-r--r-- | .gitlab-ci.yml | 509 |
1 files changed, 405 insertions, 104 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 84976328f8..748126f532 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,17 +1,31 @@ variables: GIT_SSL_NO_VERIFY: "1" + # Commit of ghc/ci-images repository from which to pull Docker images + DOCKER_REV: cefaee3c742af193e0f7783f87edb0d35374515c + before_script: - python3 .gitlab/fix-submodules.py - git submodule sync --recursive - git submodule update --init --recursive - git checkout .gitmodules + - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" stages: - - lint - - build - - full-build - - cleanup # See Note [Cleanup on Windows] + - lint # Source linting + - build # A quick smoke-test to weed out broken commits + - full-build # Build all the things + - cleanup # See Note [Cleanup on Windows] + - packaging # Source distribution, etc. + - hackage # head.hackage testing + - deploy # push documentation + +.only-default: &only-default + only: + - master + - /ghc-[0-9]+\.[0-9]+/ + - merge_requests + - tags ############################################################ # Runner Tags @@ -30,44 +44,106 @@ stages: ############################################################ ghc-linters: + allow_failure: true stage: lint - image: ghcci/linters:0.1 + image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" script: - - | - if [ -n "$CI_MERGE_REQUEST_ID" ]; then - base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)" - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA) - submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA) - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA) - .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA - .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA - fi + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)" + - "echo Merge base $base" + # - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA) + - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA) + - .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA + - .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA + dependencies: [] + tags: + - lint + only: + refs: + - merge_requests + +# We allow the submodule checker to fail when run on merge requests (to +# accomodate, e.g., haddock changes not yet upstream) but not on `master` or +# Marge jobs. +.lint-submods: + stage: lint + image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" + script: + - submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA) + dependencies: [] + tags: + - lint + +lint-submods: + extends: .lint-submods + only: + refs: + - master + - /ghc-[0-9]+\.[0-9]+/ + - wip/marge_bot_batch_merge_job + +lint-submods-mr: + extends: .lint-submods + allow_failure: true + only: + refs: + - merge_requests + +.lint-changelogs: + stage: lint + image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" dependencies: [] tags: - lint + script: + - | + grep TBA libraries/*/changelog.md && ( + echo "Error: Found \"TBA\"s in changelogs." + exit 1 + ) + +lint-changelogs: + extends: .lint-changelogs + allow_failure: true + only: + refs: + - /ghc-[0-9]+\.[0-9]+/ + +lint-release-changelogs: + extends: .lint-changelogs + only: + - tags + ############################################################ # Validation via Pipelines (hadrian) ############################################################ .validate-hadrian: + <<: *only-default allow_failure: true script: + - cabal update - git clean -xdf && git submodule foreach git clean -xdf - bash .circleci/prepare-system.sh - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi - ./boot - ./configure $CONFIGURE_ARGS - - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` + - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist + - mv _build/bindist/ghc*.tar.xz ghc.tar.xz cache: key: hadrian paths: - cabal-cache + artifacts: + when: always + paths: + - ghc.tar.xz validate-x86_64-linux-deb8-hadrian: extends: .validate-hadrian stage: build - image: ghcci/x86_64-linux-deb8:0.1 + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV" before_script: # workaround for docker permissions - sudo chown ghc:ghc -R . @@ -75,14 +151,17 @@ validate-x86_64-linux-deb8-hadrian: - git submodule sync --recursive - git submodule update --init --recursive - git checkout .gitmodules + - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" tags: - x86_64-linux + ############################################################ # Validation via Pipelines (make) ############################################################ .validate: + <<: *only-default variables: TEST_TYPE: test before_script: @@ -92,22 +171,25 @@ validate-x86_64-linux-deb8-hadrian: - ./configure $CONFIGURE_ARGS - | THREADS=`mk/detect-cpu-count.sh` - make V=0 -j$THREADS + make V=0 -j$THREADS WERROR=-Werror - | make binary-dist TAR_COMP_OPTS="-1" - mv ghc-*.tar.xz ghc.tar.xz - | THREADS=`mk/detect-cpu-count.sh` - make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml + make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE dependencies: [] artifacts: reports: junit: junit.xml expire_in: 2 week paths: - - ghc.tar.xz + - ghc-*.tar.xz - junit.xml +################################# +# x86_64-darwin +################################# + validate-x86_64-darwin: extends: .validate stage: full-build @@ -115,17 +197,20 @@ validate-x86_64-darwin: - x86_64-darwin variables: GHC_VERSION: 8.6.3 + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz" MACOSX_DEPLOYMENT_TARGET: "10.7" # Only Sierra and onwards supports clock_gettime. See #12858 ac_cv_func_clock_gettime: "no" LANG: "en_US.UTF-8" CONFIGURE_ARGS: --with-intree-gmp + TEST_ENV: "x86_64-darwin" before_script: - git clean -xdf && git submodule foreach git clean -xdf - python3 .gitlab/fix-submodules.py - git submodule sync --recursive - git submodule update --init --recursive - git checkout .gitmodules + - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" - bash .gitlab/darwin-init.sh - PATH="`pwd`/toolchain/bin:$PATH" @@ -150,6 +235,12 @@ validate-x86_64-darwin: - git submodule sync --recursive - git submodule update --init --recursive - git checkout .gitmodules + - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" + # Build hyperlinked sources for documentation when building releases + - | + if [[ -n "$CI_COMMIT_TAG" ]]; then + echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk + fi - bash .circleci/prepare-system.sh # workaround for docker permissions @@ -162,22 +253,31 @@ validate-x86_64-darwin: - cabal-cache - toolchain -validate-aarch64-linux-deb9: +################################# +# aarch64-linux-deb9 +################################# + +.build-aarch64-linux-deb9: extends: .validate-linux stage: full-build - image: ghcci/aarch64-linux-deb9:0.1 + image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV" allow_failure: true - artifacts: - when: always - expire_in: 2 week + variables: + TEST_ENV: "aarch64-linux-deb9" + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz" cache: key: linux-aarch64-deb9 tags: - aarch64-linux +validate-aarch64-linux-deb9: + extends: .build-aarch64-linux-deb9 + artifacts: + when: always + expire_in: 2 week + nightly-aarch64-linux-deb9: - extends: validate-aarch64-linux-deb9 - stage: full-build + extends: .build-aarch64-linux-deb9 artifacts: expire_in: 2 year variables: @@ -186,22 +286,28 @@ nightly-aarch64-linux-deb9: variables: - $NIGHTLY -validate-i386-linux-deb9: +################################# +# i386-linux-deb9 +################################# + +.build-i386-linux-deb9: extends: .validate-linux stage: full-build - image: ghcci/i386-linux-deb9:0.1 - allow_failure: true + image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV" + variables: + TEST_ENV: "i386-linux-deb9" + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz" + cache: + key: linux-i386-deb9 + +validate-i386-linux-deb9: + extends: .build-i386-linux-deb9 artifacts: when: always expire_in: 2 week - cache: - key: linux-i386-deb9 nightly-i386-linux-deb9: - extends: .validate-linux - stage: full-build - image: ghcci/i386-linux-deb9:0.1 - allow_failure: true + extends: .build-i386-linux-deb9 variables: TEST_TYPE: slowtest artifacts: @@ -210,22 +316,29 @@ nightly-i386-linux-deb9: only: variables: - $NIGHTLY + +################################# +# x86_64-linux-deb9 +################################# + +.build-x86_64-linux-deb9: + extends: .validate-linux + stage: full-build + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV" + variables: + TEST_ENV: "x86_64-linux-deb9" + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz" cache: - key: linux-i386-deb9 + key: linux-x86_64-deb9 validate-x86_64-linux-deb9: - extends: .validate-linux - stage: build - image: ghcci/x86_64-linux-deb9:0.2 + extends: .build-x86_64-linux-deb9 artifacts: when: always expire_in: 2 week - cache: - key: linux-x86_64-deb9 nightly-x86_64-linux-deb9: - extends: validate-x86_64-linux-deb9 - stage: build + extends: .build-x86_64-linux-deb9 artifacts: expire_in: 2 year variables: @@ -234,70 +347,93 @@ nightly-x86_64-linux-deb9: variables: - $NIGHTLY +# N.B. Has DEBUG assertions enabled in stage2 +validate-x86_64-linux-deb9-debug: + extends: .build-x86_64-linux-deb9 + stage: build + variables: + BUILD_FLAVOUR: validate + TEST_ENV: "x86_64-linux-deb9-debug" + validate-x86_64-linux-deb9-llvm: - extends: .validate-linux + extends: .build-x86_64-linux-deb9 stage: full-build - allow_failure: true - image: ghcci/x86_64-linux-deb9:0.2 variables: BUILD_FLAVOUR: perf-llvm - cache: - key: linux-x86_64-deb9 - -validate-x86_64-linux-deb8: - extends: .validate-linux - stage: full-build - image: ghcci/x86_64-linux-deb8:0.1 - cache: - key: linux-x86_64-deb8 - artifacts: - when: always - expire_in: 2 week - -validate-x86_64-linux-fedora27: - extends: .validate-linux - stage: full-build - image: ghcci/x86_64-linux-fedora27:0.1 - cache: - key: linux-x86_64-fedora27 - artifacts: - when: always - expire_in: 2 week + TEST_ENV: "x86_64-linux-deb9-llvm" validate-x86_64-linux-deb9-integer-simple: - extends: .validate-linux + extends: .build-x86_64-linux-deb9 stage: full-build variables: INTEGER_LIBRARY: integer-simple - image: ghcci/x86_64-linux-deb9:0.2 - cache: - key: linux-x86_64-deb9 + TEST_ENV: "x86_64-linux-deb9-integer-simple" + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz" nightly-x86_64-linux-deb9-integer-simple: - extends: validate-x86_64-linux-deb9-integer-simple + extends: .build-x86_64-linux-deb9 stage: full-build - artifacts: - expire_in: 2 year variables: + INTEGER_LIBRARY: integer-simple + TEST_ENV: "x86_64-linux-deb9-integer-simple" TEST_TYPE: slowtest + artifacts: + expire_in: 2 year only: variables: - $NIGHTLY validate-x86_64-linux-deb9-unreg: - extends: .validate-linux + extends: .build-x86_64-linux-deb9 stage: full-build variables: CONFIGURE_ARGS: --enable-unregisterised - image: ghcci/x86_64-linux-deb9:0.2 + TEST_ENV: "x86_64-linux-deb9-unreg" + + +################################# +# x86_64-linux-deb8 +################################# + +release-x86_64-linux-deb8: + extends: .validate-linux + stage: full-build + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV" + variables: + TEST_ENV: "x86_64-linux-deb8" + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz" + only: + - tags cache: - key: linux-x86_64-deb9 + key: linux-x86_64-deb8 + artifacts: + when: always + expire_in: 2 week + + +################################# +# x86_64-linux-fedora27 +################################# + +validate-x86_64-linux-fedora27: + extends: .validate-linux + stage: full-build + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV" + variables: + TEST_ENV: "x86_64-linux-fedora27" + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz" + cache: + key: linux-x86_64-fedora27 + artifacts: + when: always + expire_in: 2 week ############################################################ # Validation via Pipelines (Windows) ############################################################ -.validate-windows: +.build-windows: + <<: *only-default before_script: - git clean -xdf - git submodule foreach git clean -xdf @@ -314,72 +450,131 @@ validate-x86_64-linux-deb9-unreg: - git submodule sync --recursive - git submodule update --init --recursive - git checkout .gitmodules + - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true" - bash .gitlab/win32-init.sh after_script: - rd /s /q tmp - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache - bash -c 'make clean || true' dependencies: [] + variables: + FORCE_SYMLINKS: 1 + LANG: "en_US.UTF-8" cache: paths: - cabal-cache - ghc-8.6.2 - ghc-tarballs -validate-x86_64-windows-hadrian: - extends: .validate-windows +.build-windows-hadrian: + extends: .build-windows stage: full-build variables: GHC_VERSION: "8.6.2" - LANG: "en_US.UTF-8" script: - | - set MSYSTEM=MINGW64 python boot bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex' - mkdir -p _build - cp -R inplace/mingw _build/mingw - - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick" - # FIXME: Bindist disabled due to #16073 - #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist" + - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist" + - mv _build/bindist/ghc*.tar.xz ghc.tar.xz # FIXME: Testsuite disabled due to #16156. - #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml' - cache: - key: x86_64-windows + # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml' tags: - x86_64-windows + artifacts: + when: always + paths: + - ghc.tar.xz -validate-x86_64-windows: - extends: .validate-windows +validate-x86_64-windows-hadrian: + extends: .build-windows-hadrian + variables: + MSYSTEM: MINGW64 + cache: + key: x86_64-windows-hadrian + +nightly-i386-windows-hadrian: + extends: .build-windows-hadrian + variables: + MSYSTEM: MINGW32 + only: + variables: + - $NIGHTLY + cache: + key: i386-windows-hadrian + +.build-windows-make: + extends: .build-windows stage: full-build # due to #16084 allow_failure: true variables: + BUILD_FLAVOUR: "quick" GHC_VERSION: "8.6.2" - LANG: "en_US.UTF-8" + BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz" script: - | - set MSYSTEM=MINGW64 python boot - bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex' - - bash -c "echo include mk/flavours/quick.mk > mk/build.mk" + bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex $CONFIGURE_ARGS' + - bash -c "echo include mk/flavours/${BUILD_FLAVOUR}.mk > mk/build.mk" - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`" - - | - bash -c "make binary-dist TAR_COMP_OPTS=-1" - mv ghc-*.tar.xz ghc.tar.xz + - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1" - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml' - cache: - key: x86_64-windows tags: - x86_64-windows artifacts: when: always + expire_in: 2 week reports: junit: junit.xml paths: - - ghc.tar.xz + - ghc-*.tar.xz - junit.xml +validate-x86_64-windows: + extends: .build-windows-make + variables: + MSYSTEM: MINGW64 + CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32" + cache: + key: x86_64-windows + +# Normal Windows validate builds are profiled; that won't do for releases. +release-x86_64-windows: + extends: validate-x86_64-windows + variables: + MSYSTEM: MINGW64 + BUILD_FLAVOUR: "perf" + CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32" + only: + - tags + +release-i386-windows: + extends: .build-windows-make + only: + - tags + variables: + MSYSTEM: MINGW32 + BUILD_FLAVOUR: "perf" + CONFIGURE_ARGS: "--target=i386-unknown-mingw32" + cache: + key: i386-windows + +nightly-i386-windows: + extends: .build-windows-make + only: + variables: + - $NIGHTLY + variables: + MSYSTEM: MINGW32 + CONFIGURE_ARGS: "--target=i386-unknown-mingw32" + cache: + key: i386-windows + +############################################################ +# Cleanup +############################################################ + # Note [Cleaning up after shell executor] # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # @@ -393,6 +588,7 @@ validate-x86_64-windows: # See Note [Cleanup after shell executor] cleanup-windows: + <<: *only-default stage: cleanup tags: - x86_64-windows @@ -415,10 +611,12 @@ cleanup-windows: # See Note [Cleanup after shell executor] cleanup-darwin: + <<: *only-default stage: cleanup tags: - x86_64-darwin when: always + dependencies: [] before_script: - echo "Time to clean up" script: @@ -430,3 +628,106 @@ cleanup-darwin: - rm -Rf $BUILD_DIR/* - exit 0 +############################################################ +# Packaging +############################################################ + +doc-tarball: + <<: *only-default + stage: packaging + tags: + - x86_64-linux + image: ghcci/x86_64-linux-deb9:0.2 + dependencies: + - validate-x86_64-linux-deb9 + - validate-x86_64-windows + artifacts: + paths: + - haddock.html.tar.xz + - libraries.html.tar.xz + - users_guide.html.tar.xz + - index.html + - "*.pdf" + script: + - rm -Rf docs + - bash -ex distrib/mkDocs/mkDocs ghc-x86_64-deb9-linux.tar.xz ghc-x86_64-mingw32.tar.xz + - ls -lh + - mv docs/*.tar.xz docs/index.html . + +source-tarball: + stage: packaging + tags: + - x86_64-linux + image: ghcci/x86_64-linux-deb9:0.2 + only: + - tags + artifacts: + paths: + - ghc-*.tar.xz + - version + script: + - mk/get-win32-tarballs.sh download all + - ./boot + - ./configure + - make sdist + - mv sdistprep/*.xz . + - make show VALUE=version > version + + +############################################################ +# Testing via head.hackage +############################################################ + +# Triggering jobs in the ghc/head.hackage project requires that we have a job +# token for that repository. Furthermore the head.hackage CI job must have +# access to an unprivileged access token with the ability to query the ghc/ghc +# project such that it can find the job ID of the fedora27 job for the current +# pipeline. + +.hackage: + <<: *only-default + stage: hackage + image: ghcci/x86_64-linux-deb9:0.2 + tags: + - x86_64-linux + dependencies: [] + variables: + HEAD_HACKAGE_PROJECT_ID: "78" + script: + - bash .gitlab/start-head.hackage.sh + +hackage: + extends: .hackage + when: manual + +hackage-label: + extends: .hackage + only: + variables: + - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/ + +nightly-hackage: + extends: .hackage + only: + variables: + - $NIGHTLY + +pages: + stage: deploy + dependencies: + - doc-tarball + image: ghcci/x86_64-linux-deb9:0.2 + tags: + - x86_64-linux + script: + - mkdir -p public/doc + - tar -xf haddock.html.tar.xz -C public/doc + - tar -xf libraries.html.tar.xz -C public/doc + - tar -xf users_guide.html.tar.xz -C public/doc + - cp -f index.html public/doc + only: + - master + artifacts: + paths: + - public + |