variables: GIT_SSL_NO_VERIFY: "1" # Commit of ghc/ci-images repository from which to pull Docker images DOCKER_REV: 885dfba7f74583d6a394dbf711cc805178cfcc47 # Sequential version number of all cached things. # Bump to invalidate GitLab CI cache. CACHE_REV: 6 # Disable shallow clones; they break our linting rules GIT_DEPTH: 0 # Always start with a fresh clone to avoid non-hermetic builds GIT_STRATEGY: clone # Overridden by individual jobs CONFIGURE_ARGS: "" GIT_SUBMODULE_STRATEGY: "recursive" # Makes ci.sh isolate CABAL_DIR HERMETIC: "YES" # Reduce XZ compression level for regular jobs (it is bumped to 9 for releases # and nightly jobs). In my experiments I've got the following bindist size in # the given time for each compression level (with the quick flavour): # # XZ_OPT Time Size # -9 4m06s 112 MB # -8 4m00s 114 MB # -7 3m50s 116 MB # -6 (default) 3m40s 118 MB # -5 2m47s 123 MB # -4 1m57s 134 MB # -3 1m03s 129 MB # -2 49.73s 136 MB # -1 37.72s 142 MB # -0 34.40s 156 MB # XZ_OPT: "-1" default: interruptible: true stages: - not-interruptible - tool-lint # Source linting of the tools - quick-build # A very quick smoke-test to weed out broken commits - full-build # Build all the things - packaging # Source distribution, etc. - testing # head.hackage correctness and compiler performance testing - deploy # push documentation # Note [The CI Story] # ~~~~~~~~~~~~~~~~~~~ # # There are two different types of pipelines: # # - marge-bot merges to `master`. Here we perform an exhaustive validation # across all of the platforms which we support. In addition, we push # performance metric notes upstream, providing a persistent record of the # performance characteristics of the compiler. # # - merge requests. Here we perform a slightly less exhaustive battery of # testing. Namely we omit some configurations (e.g. the unregisterised job). # These use the merge request's base commit for performance metric # comparisons. # workflow: # N.B. Don't run on wip/ branches, instead on run on merge requests. rules: - if: $CI_MERGE_REQUEST_ID - if: $CI_COMMIT_TAG - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH =~ /ghc-[0-9]+\.[0-9]+/' - if: '$CI_PIPELINE_SOURCE == "web"' # Allow linters to fail on draft MRs. # This must be explicitly transcluded in lint jobs which # override `rules:` .drafts-can-fail-lint: &drafts-can-fail-lint if: "$CI_MERGE_REQUEST_TITLE =~ /^\\s*(Draft|wip|WIP):/" allow_failure: true .lint: stage: tool-lint tags: - lint rules: - *drafts-can-fail-lint - when: always .nightly: &nightly variables: XZ_OPT: "-9" rules: - if: $NIGHTLY artifacts: when: always expire_in: 8 weeks .release: &release variables: BUILD_FLAVOUR: "perf" XZ_OPT: "-9" IGNORE_PERF_FAILURES: "all" HADDOCK_HYPERLINKED_SOURCES: "YES" artifacts: when: always expire_in: 1 year rules: - if: '$RELEASE_JOB == "yes"' ############################################################ # Runner Tags ############################################################ # # * x86_64-linux: Any Docker-capable x86_64 Linux machine # * aarch64-linux: Any Docker-capable AArch64 Linux machine # * x86_64-windows: A x86_64 Windows machine # * lint: Any Docker-capable x86_64 Linux machine; distinct from # x86_64-linux to ensure low-latency availability. # #### # HACK ### # # Since 58cfcc65 the default for jobs has been "interruptible", this means # that when new commits are pushed to a branch which already has a running # pipeline then the old pipelines for this branch are cancelled. # # This includes the master branch, and in particular, new commits merged # to the master branch will cancel the nightly job. # # The semantics of pipeline cancelling are actually a bit more complicated # though. The interruptible flag is *per job*, but once a pipeline has run # *any* non-interruptible job, then the whole pipeline is considered # non-interruptible (ref # https://gitlab.com/gitlab-org/gitlab/-/issues/32837). This leads to the # hack in this MR where by default all jobs are `interruptible: True`, but # for pipelines we definitely want to run, there is a dummy job which # happens first, which is `interreuptible: False`. This has the effect of # dirtying the whole pipeline and # # For now, this patch solves the immediate problem of making sure nightly # jobs are not cancelled. # In the future, we may want to enable this job also for the master # branch, making that change might mean we need more CI capacity than # currently available. not-interruptible: stage: not-interruptible script: "true" interruptible: false image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" tags: - lint rules: # - if: '$CI_COMMIT_BRANCH == "master"' # when: always - if: $NIGHTLY when: always ############################################################ # Validate jobs ############################################################ # These jobs are generated by running the ./.gitlab/generate_jobs script include: '.gitlab/jobs.yaml' ############################################################ # tool linting ############################################################ ghc-linters: stage: tool-lint image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" extends: .lint-params variables: BUILD_FLAVOUR: default script: - .gitlab/ci.sh configure - timeout 10m .gitlab/ci.sh run_hadrian test --test-root-dirs="testsuite/tests/linters" dependencies: [] rules: - if: $CI_MERGE_REQUEST_ID - *drafts-can-fail-lint # Run mypy Python typechecker on linter scripts. lint-linters: image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" extends: .lint script: - mypy testsuite/tests/linters/regex-linters/*.py dependencies: [] # Check that .T files all parse by listing broken tests. lint-testsuite: image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV" extends: .lint script: - make -Ctestsuite list_broken TEST_HC=$GHC dependencies: [] # Run mypy Python typechecker on testsuite driver typecheck-testsuite: image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" extends: .lint script: - mypy testsuite/driver/runtests.py dependencies: [] # We allow the submodule checker to fail when run on merge requests (to # accommodate, e.g., haddock changes not yet upstream) but not on `master` or # Marge jobs. .lint-submods: image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" extends: .lint-params variables: BUILD_FLAVOUR: default script: - .gitlab/ci.sh configure - .gitlab/ci.sh run_hadrian stage1:exe:lint-submodule-refs - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)" - "echo Linting submodule changes between $base..$CI_COMMIT_SHA" - git submodule foreach git remote update - _build/stage0/bin/lint-submodule-refs . $(git rev-list $base..$CI_COMMIT_SHA) dependencies: [] lint-submods: extends: .lint-submods # Allow failure on merge requests since any necessary submodule patches may # not be upstreamed yet. rules: - if: '$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/' allow_failure: false - allow_failure: true lint-submods-branch: extends: .lint-submods variables: BUILD_FLAVOUR: default script: - .gitlab/ci.sh configure - .gitlab/ci.sh run_hadrian stage1:exe:lint-submodule-refs - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA" - git submodule foreach git remote update - _build/stage0/bin/lint-submodule-refs . $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA) rules: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/' - *drafts-can-fail-lint ############################################################ # GHC source code linting ############################################################ .lint-params: needs: [] image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" extends: .lint before_script: - export PATH="/opt/toolchain/bin:$PATH" # workaround for docker permissions - sudo chown ghc:ghc -R . - .gitlab/ci.sh setup after_script: - .gitlab/ci.sh save_cache - cat ci-timings variables: GHC_FLAGS: -Werror cache: key: lint-$CACHE_REV paths: - cabal-cache hlint-ghc-and-base: extends: .lint-params image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" variables: BUILD_FLAVOUR: default script: - .gitlab/ci.sh setup - .gitlab/ci.sh configure - .gitlab/ci.sh run_hadrian lint:base - .gitlab/ci.sh run_hadrian lint:compiler ############################################################ # GHC-in-GHCi (Hadrian) ############################################################ hadrian-ghc-in-ghci: stage: quick-build needs: [lint-linters, lint-submods] image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" before_script: # workaround for docker permissions - sudo chown ghc:ghc -R . variables: GHC_FLAGS: -Werror tags: - x86_64-linux script: - git clean -xdf && git submodule foreach git clean -xdf - .gitlab/ci.sh setup - .gitlab/ci.sh configure # Load ghc-in-ghci then immediately exit and check the modules loaded - echo ":q" | hadrian/ghci -j`mk/detect-cpu-count.sh`| tail -n2 | grep "Ok," after_script: - .gitlab/ci.sh save_cache - cat ci-timings cache: key: hadrian-ghci-$CACHE_REV paths: - cabal-cache ############################################################ # stack-hadrian-build ############################################################ # Verify that Hadrian builds with stack. Note that we don't actually perform a # build of GHC itself; we merely test that the Hadrian executable builds and # works (by invoking `hadrian --version`). stack-hadrian-build: extends: hadrian-ghc-in-ghci stage: quick-build script: - .gitlab/ci.sh setup - .gitlab/ci.sh configure - hadrian/build-stack --version ################################# # x86_64-linux-deb10-make ################################# # One job still using Make for now but we don't produce any release bindists with it. .build-x86_64-linux-deb10-make: extends: .validate image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" variables: TEST_ENV: "x86_64-linux-deb10-make" BUILD_FLAVOUR: "validate" BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb10-linux.tar.gz" before_script: # workaround for docker permissions - sudo chown ghc:ghc -R . after_script: - .gitlab/ci.sh save_cache - .gitlab/ci.sh clean - cat ci-timings tags: - x86_64-linux validate-x86_64-linux-deb10-make: extends: .build-x86_64-linux-deb10-make stage: full-build rules: - if: '$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/' ############################################################ # Validation via Pipelines (make) ############################################################ .validate: variables: TEST_TYPE: test MAKE_ARGS: "-Werror" script: | # Build hyperlinked sources for documentation when building releases if [[ "$RELEASE_JOB" = "yes" ]]; then HADDOCK_HYPERLINKED_SOURCES=1 fi .gitlab/ci.sh setup .gitlab/ci.sh configure .gitlab/ci.sh build_make .gitlab/ci.sh test_make dependencies: [] artifacts: reports: junit: junit.xml expire_in: 2 week paths: - $BIN_DIST_PREP_TAR_COMP - junit.xml - performance-metrics.tsv #################################### # Testing reinstallable ghc codepath #################################### test-cabal-reinstall-x86_64-linux-deb10: extends: nightly-x86_64-linux-deb10-validate stage: full-build before_script: - ./boot - ./configure - "cabal update --index=$HACKAGE_INDEX_STATE --project-file=cabal.project-reinstall" variables: REINSTALL_GHC: "yes" BUILD_FLAVOUR: validate TEST_ENV: "x86_64-linux-deb10-cabal-install" rules: - if: $NIGHTLY ############################################################ # Packaging ############################################################ doc-tarball: stage: packaging needs: - job: x86_64-linux-deb10-numa-slow-validate optional: true - job: nightly-x86_64-linux-deb10-validate optional: true - job: release-x86_64-linux-deb10-perf optional: true - job: x86_64-windows-validate optional: true - job: nightly-x86_64-windows-validate optional: true - job: release-x86_64-windows-perf optional: true tags: - x86_64-linux image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" dependencies: null variables: LINUX_BINDIST: "ghc-x86_64-linux-deb10.tar.xz" WINDOWS_BINDIST: "ghc-x86_64-windows.tar.xz" artifacts: paths: - haddock.html.tar.xz - libraries.html.tar.xz - users_guide.html.tar.xz - index.html - "*.pdf" script: - | mv "ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz" "$LINUX_BINDIST" \ || mv "ghc-x86_64-linux-deb10-validate.tar.xz" "$LINUX_BINDIST" \ || mv "ghc-x86_64-linux-deb10-perf.tar.xz" "$LINUX_BINDIST" \ || true mv "ghc-x86_64-windows-validate.tar.xz" "$WINDOWS_BINDIST" \ || mv "ghc-x86-windows-perf.tar.xz" "$WINDOWS_BINDIST" \ || true if [ ! -f "$LINUX_BINDIST" ]; then echo "Error: $LINUX_BINDIST does not exist. Did the Debian 9 job fail?" exit 1 fi if [ ! -f "$WINDOWS_BINDIST" ]; then echo "Error: $WINDOWS_BINDIST does not exist. Did the 64-bit Windows job fail?" exit 1 fi - rm -Rf docs - bash -ex distrib/mkDocs/mkDocs $LINUX_BINDIST $WINDOWS_BINDIST - ls -lh - mv docs/*.tar.xz docs/index.html . source-tarball: stage: full-build tags: - x86_64-linux image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" dependencies: [] artifacts: paths: - ghc-*.tar.xz - hadrian-bootstrap-sources-*.tar.gz script: - ./boot - ./configure - ./hadrian/build source-dist - mv _build/source-dist/*.xz . - python3 ./hadrian/bootstrap/bootstrap.py -w $GHC fetch -o hadrian-bootstrap-sources-$GHC_VERSION rules: - if: $NIGHTLY test-bootstrap: stage: full-build needs: [source-tarball] tags: - x86_64-linux image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" dependencies: null script: - mkdir test-bootstrap - tar -xf ghc-*[0-9]-src.tar.xz -C test-bootstrap - tar -xf ghc-*-testsuite.tar.xz -C test-bootstrap - cp hadrian-bootstrap-sources-*.tar.gz test-bootstrap/ghc-* - pushd test-bootstrap/ghc-* - python3 ./hadrian/bootstrap/bootstrap.py -w $GHC --bootstrap-sources hadrian-bootstrap-sources-*.tar.gz - export HADRIAN_PATH="$PWD/_build/bin/hadrian" - .gitlab/ci.sh setup - .gitlab/ci.sh configure - .gitlab/ci.sh build_hadrian - .gitlab/ci.sh test_hadrian - popd - rm -Rf test-bootstrap variables: TEST_ENV: "x86_64-linux-deb10-hadrian" BIN_DIST_NAME: "ghc-x86_64-deb10-linux" BUILD_FLAVOUR: "validate" rules: - if: $NIGHTLY ############################################################ # Testing via head.hackage ############################################################ # Triggering jobs in the ghc/head.hackage project requires that we have a job # token for that repository. Furthermore the head.hackage CI job must have # access to an unprivileged access token with the ability to query the ghc/ghc # project such that it can find the job ID of the fedora33 job for the current # pipeline. .hackage: stage: testing needs: [doc-tarball] variables: UPSTREAM_PROJECT_PATH: "$CI_PROJECT_PATH" UPSTREAM_PROJECT_ID: "$CI_PROJECT_ID" UPSTREAM_PIPELINE_ID: "$CI_PIPELINE_ID" trigger: project: "ghc/head.hackage" branch: "master" strategy: "depend" hackage-lint: extends: .hackage variables: EXTRA_HC_OPTS: "-dcore-lint" when: manual hackage-label-lint: extends: .hackage variables: EXTRA_HC_OPTS: "-dcore-lint" rules: - if: '$CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/' # The head.hackage job is split into two jobs because enabling `-dcore-lint` # affects the total allocation numbers for the simplifier portion significantly. nightly-hackage-lint: rules: - if: $NIGHTLY extends: .hackage variables: EXTRA_HC_OPTS: "-dcore-lint" nightly-hackage-perf: rules: - if: $NIGHTLY extends: .hackage variables: # Generate logs for nightly builds which include timing information. EXTRA_HC_OPTS: "-ddump-timings" ############################################################ # Nofib testing ############################################################ perf-nofib: # Dependencies used by perf-nofib can't be built when some compiler changes # aren't (yet) supported by head.hackage. # Hence we allow this job to fail. allow_failure: true stage: testing needs: - job: x86_64-linux-fedora33-perf optional: true - job: nightly-x86_64-linux-fedora33-perf optional: true - job: release-x86_64-linux-fedora33-perf optional: true - job: doc-tarball optional: false image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora33:$DOCKER_REV" rules: - if: $CI_MERGE_REQUEST_ID - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/' tags: - x86_64-linux before_script: - cd nofib - "cabal update --index=$HACKAGE_INDEX_STATE --project-file=cabal.project.head-hackage" script: - root=$(pwd)/ghc - | mkdir tmp tar -xf ../ghc-x86_64-linux-fedora33-perf.tar.xz -C tmp pushd tmp/ghc-*/ ./configure --prefix=$root make install popd rm -Rf tmp - export PATH=$root/bin:$PATH - cabal install -w "$root/bin/ghc" --lib regex-compat unboxed-ref parallel random-1.2.1 --allow-newer --package-env local.env --project-file=cabal.project.head-hackage - export GHC_ENVIRONMENT="$(pwd)/local.env" - "make HC=$root/bin/ghc BOOT_HC=$root/bin/ghc boot mode=fast -j$CPUS" - "make HC=$root/bin/ghc BOOT_HC=$root/bin/ghc EXTRA_RUNTEST_OPTS='-cachegrind +RTS -V0 -RTS' NoFibRuns=1 mode=fast -j$CPUS 2>&1 | tee nofib.log" artifacts: expire_in: 12 week when: always paths: - nofib/nofib.log rules: - if: '$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/' ############################################################ # Ad-hoc performance testing ############################################################ perf: stage: testing needs: - job: x86_64-linux-fedora33-perf optional: true - job: nightly-x86_64-linux-fedora33-perf optional: true - job: release-x86_64-linux-fedora33-perf optional: true - job: doc-tarball optional: false dependencies: null image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora33:$DOCKER_REV" rules: - if: $CI_MERGE_REQUEST_ID - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/' tags: - x86_64-linux-perf script: - root=$(pwd)/ghc - | mkdir tmp tar -xf ghc-x86_64-linux-fedora33-perf.tar.xz -C tmp pushd tmp/ghc-*/ ./configure --prefix=$root make install popd rm -Rf tmp - export BOOT_HC=$(which ghc) - export HC=$root/bin/ghc - .gitlab/ci.sh perf_test artifacts: expire_in: 2 year when: always paths: - out rules: - if: '$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/' ############################################################ # Documentation deployment via GitLab Pages ############################################################ pages: stage: deploy needs: [doc-tarball] dependencies: null image: ghcci/x86_64-linux-deb9:0.2 # See #18973 allow_failure: true tags: - x86_64-linux script: - mkdir -p public/doc - tar -xf haddock.html.tar.xz -C public/doc - tar -xf libraries.html.tar.xz -C public/doc - tar -xf users_guide.html.tar.xz -C public/doc - | cat >public/index.html < EOF - cp -f index.html public/doc rules: # N.B. only run this on ghc/ghc since the deployed pages are quite large # and we only serve GitLab Pages for ghc/ghc. - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_NAMESPACE == "ghc"' artifacts: paths: - public