diff options
author | Ben Gamari <ben@smart-cactus.org> | 2018-12-26 12:01:10 -0500 |
---|---|---|
committer | Ben Gamari <ben@smart-cactus.org> | 2018-12-27 01:11:20 -0500 |
commit | b6237131cd982ff0dad350422a0b8245a3d63633 (patch) | |
tree | 1804616bab88b0817eaf0d65f6bfa623e12815f5 | |
parent | c68270119b7f215a6d7f30fc020856fa94056dc3 (diff) | |
download | haskell-b6237131cd982ff0dad350422a0b8245a3d63633.tar.gz |
Grab CI configuration from master
This is awfully ugly but is nevertheless significantly less error-prone
than cherry-picking all of the relevant commits manually.
23 files changed, 1457 insertions, 109 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 6dcb218770..31b64a1c22 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,21 +15,22 @@ aliases: name: submodules command: .circleci/fetch-submodules.sh - &buildenv - THREADS: 9 - SKIP_PERF_TESTS: YES + # ideally we would simply set THREADS here instead of re-detecting it every + # time we need it below. Unfortunately, there is no way to set an environment + # variable with the result of a shell script. + SKIP_PERF_TESTS: NO VERBOSE: 2 - &boot run: name: Boot command: | ./boot - - cat <<EOF >> mk/build.mk - BuildFlavour=$BUILD_FLAVOUR - ifneq "\$(BuildFlavour)" "" - include mk/flavours/\$(BuildFlavour).mk - endif - EOF + - &set_git_identity + run: + name: Set Git Identity + command: | + git config user.email "ghc-circleci@haskell.org" + git config user.name "GHC CircleCI" - &configure_unix run: name: Configure @@ -50,131 +51,197 @@ aliases: - &make run: name: Build - command: "make -j$THREADS V=0" + command: "make -j`mk/detect-cpu-count.sh` V=0" - &build_hadrian run: name: Build GHC using Hadrian command: | cabal update - hadrian/build.sh -j$THREADS + hadrian/build.sh -j`mk/detect-cpu-count.sh` - &test run: name: Test command: | mkdir -p test-results - make test SKIP_PERF_TESTS=YES JUNIT_FILE=../../test-results/junit.xml + METRICS_FILE=$(mktemp) + echo "export METRICS_FILE=$METRICS_FILE" >> $BASH_ENV + make test THREADS=`mk/detect-cpu-count.sh` SKIP_PERF_TESTS=$SKIP_PERF_TESTS TEST_ENV=$TEST_ENV JUNIT_FILE=../../test-results/junit.xml METRICS_FILE=$METRICS_FILE - &store_test_results store_test_results: path: test-results + - &push_perf_note + run: + name: Push Performance Git Notes + command: .circleci/push-test-metrics.sh + - &store_test_artifacts + store_artifacts: + # we might want to add the whole output of the test phase + # too at some point + path: test-results/junit.xml - &slowtest run: name: Full Test - command: make slowtest SKIP_PERF_TESTS=YES + command: | + mkdir -p test-results + make slowtest SKIP_PERF_TESTS=YES THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../test-results/junit.xml - &bindist run: name: Create bindist # Reduce compression effort to 3 - command: make binary-dist TAR_COMP_OPTS="-2" + command: make binary-dist TAR_COMP_OPTS="-2" && mv ghc*.tar.xz ghc.tar.xz # Building bindist takes ~15 minutes without output, account for # that. no_output_timeout: "30m" - - &storeartifacts - run: - name: Store artifacts - command: | - curl https://ghc-artifacts.s3.amazonaws.com/tools/ghc-artifact-collector-$GHC_COLLECTOR_FLAVOR --output ghc-artifact-collector - chmod +x ghc-artifact-collector - ./ghc-artifact-collector ghc*.tar.xz - - &trigger_on_tags - filters: - tags: - only: /^ghc-.*/ + - &store_bindist + store_artifacts: + path: ghc.tar.xz + - &only_release_tags + tags: + only: /^ghc-.*/ + - &ignore_gitlab_branches + branches: + ignore: /^gitlab\/.*/ jobs: - "validate-x86_64-linux": - resource_class: xlarge + "validate-x86_64-freebsd": + docker: + - image: ghcci/x86_64-freebsd + environment: + TARGET: FreeBSD + <<: *buildenv + GHC_COLLECTOR_FLAVOR: x86_64-freebsd + TEST_ENV: x86_64-freebsd + steps: + - checkout + - *set_git_identity + - *prepare + - *submodules + - *boot + - *configure_bsd + - *make + - *bindist + - *store_bindist + - *test + - *store_test_results + - *store_test_artifacts + - *push_perf_note + + "validate-x86_64-darwin": + macos: + xcode: "9.0" + environment: + # Disable sphinx PDF output as MacTeX apparently doesn't provide xelatex + BUILD_SPHINX_PDF: "NO" + MACOSX_DEPLOYMENT_TARGET: "10.7" + # Only Sierra and onwards supports clock_gettime. See #12858 + ac_cv_func_clock_gettime: "no" + GHC_COLLECTOR_FLAVOR: x86_64-darwin + <<: *buildenv + TEST_ENV: x86_64-darwin + steps: + - checkout + - *set_git_identity + - *prepare + - *submodules + - *boot + - *configure_unix + - *make + - *bindist + - *store_bindist + - *test + - *store_test_results + - *store_test_artifacts + - *push_perf_note + + "validate-x86_64-linux-deb8": docker: - - image: ghcci/x86_64-linux:0.0.4 + - image: ghcci/x86_64-linux-deb8:0.1 environment: <<: *buildenv GHC_COLLECTOR_FLAVOR: x86_64-linux + TEST_ENV: x86_64-linux-deb8 steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot - *configure_unix - *make - *bindist - - *storeartifacts + - *store_bindist - *test - *store_test_results + - *store_test_artifacts + - *push_perf_note - "validate-x86_64-freebsd": - resource_class: xlarge + "validate-x86_64-linux-deb9-integer-simple": docker: - - image: ghcci/x86_64-freebsd + - image: ghcci/x86_64-linux-deb9:0.2 environment: - TARGET: FreeBSD <<: *buildenv - GHC_COLLECTOR_FLAVOR: x86_64-freebsd + INTEGER_LIBRARY: integer-simple + GHC_COLLECTOR_FLAVOR: x86_64-linux-deb9 steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot - - *configure_bsd + - *configure_unix - *make - *bindist - - *storeartifacts + - *store_bindist - *test - *store_test_results + - *store_test_artifacts + - *push_perf_note - "validate-x86_64-darwin": - macos: - xcode: "9.0" + "validate-x86_64-linux-deb9": + docker: + - image: ghcci/x86_64-linux-deb9:0.1 environment: - # Disable sphinx PDF output as MacTeX apparently doesn't provide xelatex - BUILD_SPHINX_PDF: "NO" - MACOSX_DEPLOYMENT_TARGET: "10.7" - # Only Sierra and onwards supports clock_gettime. See #12858 - ac_cv_func_clock_gettime: "no" - GHC_COLLECTOR_FLAVOR: x86_64-darwin <<: *buildenv + GHC_COLLECTOR_FLAVOR: x86_64-linux-deb9 + TEST_ENV: x86_64-linux-deb9 steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot - *configure_unix - *make - *bindist - - *storeartifacts + - *store_bindist - *test - *store_test_results + - *store_test_artifacts + - *push_perf_note - "validate-hadrian-x86_64-linux": - resource_class: xlarge + "validate-hadrian-x86_64-linux-deb8": docker: - - image: ghcci/x86_64-linux:0.0.4 + - image: ghcci/x86_64-linux-deb8:0.1 environment: <<: *buildenv steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot - *configure_unix - *build_hadrian - "validate-x86_64-linux-unreg": - resource_class: xlarge + "validate-x86_64-linux-deb8-unreg": docker: - - image: ghcci/x86_64-linux:0.0.4 + - image: ghcci/x86_64-linux-deb8:0.1 environment: <<: *buildenv + TEST_ENV: x86_64-linux-deb8-unreg steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot @@ -182,41 +249,44 @@ jobs: - *make - *test - *store_test_results + - *push_perf_note + - *store_test_artifacts - "validate-x86_64-linux-llvm": - resource_class: xlarge + "validate-x86_64-linux-deb9-llvm": docker: - - image: ghcci/x86_64-linux:0.0.4 + - image: ghcci/x86_64-linux-deb9:0.2 environment: <<: *buildenv BUILD_FLAVOUR: perf-llvm + TEST_ENV: x86_64-linux-deb9-llvm steps: - run: - name: Install LLVM - command: | - curl http://releases.llvm.org/6.0.0/clang+llvm-6.0.0-x86_64-linux-gnu-debian8.tar.xz | tar -xJC .. - echo "export PATH=`pwd`/../clang+llvm-6.0.0-x86_64-linux-gnu-debian8/bin:\$PATH" >> $BASH_ENV - - run: name: Verify that llc works command: llc - checkout + - *set_git_identity - *prepare - *submodules - *boot - *configure_unix - *make - - *slowtest + - *test + - *store_test_results + - *store_test_artifacts + - *push_perf_note # Nightly build with -DDEBUG using devel2 flavour - "validate-x86_64-linux-debug": - resource_class: xlarge + "validate-x86_64-linux-deb8-debug": docker: - - image: ghcci/x86_64-linux:0.0.4 + - image: ghcci/x86_64-linux-deb8:0.1 environment: BUILD_FLAVOUR: devel2 <<: *buildenv + TEST_ENV: x86_64-linux-deb8-debug + SKIP_PERF_TESTS: YES steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot @@ -224,52 +294,106 @@ jobs: - *make - *test - *store_test_results + - *store_test_artifacts + - *push_perf_note + + "validate-i386-linux-deb9": + docker: + - image: ghcci/i386-linux-deb9:0.2 + environment: + <<: *buildenv + GHC_COLLECTOR_FLAVOR: i386-linux-deb9 + TEST_ENV: i386-linux-deb9 + steps: + - checkout + - *set_git_identity + - *prepare + - *submodules + - *boot + - *configure_unix_32 + - *make + - *bindist + - *store_bindist + - *test + - *store_test_results + - *store_test_artifacts + - *push_perf_note - "validate-i386-linux": - resource_class: xlarge + "validate-i386-linux-deb8": docker: - - image: ghcci/i386-linux:0.0.5 + - image: ghcci/i386-linux-deb8:0.1 environment: <<: *buildenv GHC_COLLECTOR_FLAVOR: i386-linux + TEST_ENV: i386-linux-deb8 steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot - *configure_unix_32 - *make - *bindist - - *storeartifacts + - *store_bindist + - *test + - *store_test_results + - *store_test_artifacts + - *push_perf_note + + "validate-x86_64-linux-centos7": + docker: + - image: ghcci/x86_64-linux-centos7:0.1 + environment: + <<: *buildenv + GHC_COLLECTOR_FLAVOR: x86_64-centos7 + LANG: en_US.UTF-8 + TEST_ENV: x86_64-centos7 + # Sphinx is too old + BUILD_SPHINX_PDF: NO + steps: + - checkout + - *set_git_identity + - *prepare + - *submodules + - *boot + - *configure_unix + - *make + - *bindist + - *store_bindist - *test - *store_test_results + - *store_test_artifacts + - *push_perf_note - "validate-x86_64-fedora": - resource_class: xlarge + "validate-x86_64-linux-fedora27": docker: - - image: ghcci/x86_64-linux-fedora:0.0.15 + - image: ghcci/x86_64-linux-fedora27:0.1 environment: <<: *buildenv GHC_COLLECTOR_FLAVOR: x86_64-fedora + TEST_ENV: x86_64-linux-fedora27 steps: - checkout + - *set_git_identity - *prepare - *submodules - *boot - *configure_unix - *make - *bindist - - *storeartifacts + - *store_bindist - *test - *store_test_results + - *store_test_artifacts + - *push_perf_note - "slow-validate-x86_64-linux": - resource_class: xlarge + "slow-validate-x86_64-linux-deb8": docker: - - image: ghcci/x86_64-linux:0.0.4 + - image: ghcci/x86_64-linux-deb8:0.1 environment: <<: *buildenv - GHC_COLLECTOR_FLAVOR: x86_64-linux + GHC_COLLECTOR_FLAVOR: x86_64-linux-deb8 steps: - checkout - *prepare @@ -278,23 +402,43 @@ jobs: - *configure_unix - *make - *slowtest + - *store_test_results + - *store_test_artifacts + - *push_perf_note workflows: version: 2 validate: jobs: - - validate-x86_64-linux: - *trigger_on_tags + - validate-x86_64-linux-deb8: + filters: + <<: [*only_release_tags, *ignore_gitlab_branches] # FreeBSD disabled: https://github.com/haskell/unix/issues/102 # - validate-x86_64-freebsd - validate-x86_64-darwin: - *trigger_on_tags - - validate-x86_64-linux-llvm - - validate-i386-linux: - *trigger_on_tags - - validate-hadrian-x86_64-linux - - validate-x86_64-fedora: - *trigger_on_tags + filters: + <<: [*only_release_tags, *ignore_gitlab_branches] + - validate-x86_64-linux-deb8-llvm: + filters: + <<: *ignore_gitlab_branches + - validate-i386-linux-deb8: + filters: + <<: [*only_release_tags, *ignore_gitlab_branches] + - validate-x86_64-linux-deb9: + filters: + <<: [*only_release_tags, *ignore_gitlab_branches] + - validate-i386-linux-deb9: + filters: + <<: [*only_release_tags, *ignore_gitlab_branches] + - validate-x86_64-linux-centos7: + filters: + <<: [*only_release_tags, *ignore_gitlab_branches] + - validate-hadrian-x86_64-linux-deb8: + filters: + <<: *ignore_gitlab_branches + - validate-x86_64-linux-fedora27: + filters: + <<: [*only_release_tags, *ignore_gitlab_branches] nightly: triggers: @@ -305,10 +449,12 @@ workflows: only: - master jobs: - - validate-x86_64-linux-unreg - - validate-x86_64-linux-llvm - - validate-x86_64-linux-debug - - slow-validate-x86_64-linux + - validate-x86_64-linux-deb8-unreg + - validate-x86_64-linux-deb8-llvm + - validate-x86_64-linux-deb8-debug + - validate-x86_64-linux-deb9 + - validate-x86_64-linux-deb9-integer-simple + - slow-validate-x86_64-linux-deb8 notify: webhooks: diff --git a/.circleci/images/aarch64-linux-deb9/Dockerfile b/.circleci/images/aarch64-linux-deb9/Dockerfile new file mode 100644 index 0000000000..f871f75b2f --- /dev/null +++ b/.circleci/images/aarch64-linux-deb9/Dockerfile @@ -0,0 +1,69 @@ +FROM aarch64/debian:stretch + +ENV LANG C.UTF-8 + +# Core build utilities +RUN apt-get update -qq +RUN apt-get install -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \ + ca-certificates g++ git make automake autoconf gcc \ + perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo \ + jq wget curl libnuma-dev + +# Documentation tools +RUN apt-get install -qy python3-sphinx texlive-xetex texlive-latex-extra + +# Boot LLVM +ENV PATH /usr/local/bin:$PATH +ENV LLVM_TARBALL http://releases.llvm.org/6.0.0/clang+llvm-6.0.0-aarch64-linux-gnu.tar.xz +RUN curl $LLVM_TARBALL | tar -xJC .. && \ + mkdir /opt/llvm6 && \ + cp -R clang+llvm*/* /opt/llvm6 && \ + rm -R clang+llvm* && \ + /opt/llvm6/bin/llc --version + +# GHC +#RUN curl http://downloads.haskell.org/~ghc/8.6.2/ghc-8.6.2-aarch64-deb8-linux.tar.xz | tar -xJ && \ +COPY ghc-8.6.2-aarch64-unknown-linux.tar.xz . +RUN cat ghc-8.6.2-aarch64-unknown-linux.tar.xz | tar -xJ && \ + cd ghc-8.6.2 && \ + ./configure --prefix=/usr/local LLC=/opt/llvm6/bin/llc OPT=/opt/llvm6/bin/opt && \ + make install && \ + cd .. && \ + rm -Rf ghc-* +RUN ghc --version + +# LLVM +ENV LLVM_TARBALL http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-aarch64-linux-gnu.tar.xz +ENV LLC /opt/llvm7/bin/llc +ENV OPT /opt/llvm7/bin/opt +RUN curl $LLVM_TARBALL | tar -xJC .. && \ + mkdir /opt/llvm7 && \ + cp -R clang+llvm*/* /opt/llvm7 && \ + rm -R clang+llvm* && \ + $LLC --version + +# Cabal +RUN git clone https://github.com/haskell/Cabal && \ + cd Cabal && \ + git checkout cabal-install-v2.4.1.0 && \ + cd cabal-install && \ + ./bootstrap.sh --global --no-doc + +RUN ls Cabal/cabal-install/dist/build + +ENV PATH /home/ghc/.local/bin:/opt/cabal/2.2/bin:/opt/ghc/8.4.2/bin:$PATH + +# Create a normal user. +RUN adduser ghc --gecos "GHC builds" --disabled-password +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc + +# Build Haskell tools +RUN cabal update && \ + cabal install hscolour happy alex +ENV PATH /home/ghc/.cabal/bin:$PATH + +WORKDIR /home/ghc/ + +CMD ["bash"] + diff --git a/.circleci/images/i386-linux-deb8/Dockerfile b/.circleci/images/i386-linux-deb8/Dockerfile new file mode 100644 index 0000000000..c8e51810c2 --- /dev/null +++ b/.circleci/images/i386-linux-deb8/Dockerfile @@ -0,0 +1,45 @@ +FROM i386/debian:jessie + +ENV LANG C.UTF-8 + +RUN echo 'deb http://ppa.launchpad.net/hvr/ghc/ubuntu trusty main' > /etc/apt/sources.list.d/ghc.list +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6F88286 +RUN apt-get update -qq + +# Core build utilities +RUN apt-get install -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \ + ca-certificates g++ git make automake autoconf gcc \ + perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo \ + jq wget curl + +# Documentation tools +RUN apt-get install -qy python3-sphinx texlive-xetex texlive-latex-extra + +# Core build utilities +RUN apt-get install -qy libgmp-dev:i386 + +# Get i386 GHC bindist for 32 bit CI builds. +RUN cd /tmp && curl https://downloads.haskell.org/~ghc/8.4.2/ghc-8.4.2-i386-deb8-linux.tar.xz | tar -Jx +RUN cd /tmp/ghc-8.4.2 && setarch i386 ./configure --prefix=/opt/ghc-i386/8.4.2 CFLAGS=-m32 --target=i386-unknown-linux --build=i386-unknown-linux --host=i386-unknown-linux +RUN cd /tmp/ghc-8.4.2 && make install +RUN rm -rf /tmp/ghc-8.4.2 +ENV PATH /opt/ghc-i386/8.4.2/bin:$PATH + +# Get Cabal +RUN cd /tmp && \ + curl https://www.haskell.org/cabal/release/cabal-install-2.2.0.0/cabal-install-2.2.0.0-i386-unknown-linux.tar.gz | tar -zx && \ + mv cabal /usr/local/bin/cabal + +# Create a normal user. +RUN adduser ghc --gecos "GHC builds" --disabled-password +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc + +# Build Haskell tools +RUN cabal update && \ + cabal install hscolour happy alex +ENV PATH /home/ghc/.cabal/bin:$PATH + +WORKDIR /home/ghc/ + +CMD ["bash"] diff --git a/.circleci/images/i386-linux-deb9/Dockerfile b/.circleci/images/i386-linux-deb9/Dockerfile new file mode 100644 index 0000000000..c8e51810c2 --- /dev/null +++ b/.circleci/images/i386-linux-deb9/Dockerfile @@ -0,0 +1,45 @@ +FROM i386/debian:jessie + +ENV LANG C.UTF-8 + +RUN echo 'deb http://ppa.launchpad.net/hvr/ghc/ubuntu trusty main' > /etc/apt/sources.list.d/ghc.list +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6F88286 +RUN apt-get update -qq + +# Core build utilities +RUN apt-get install -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \ + ca-certificates g++ git make automake autoconf gcc \ + perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo \ + jq wget curl + +# Documentation tools +RUN apt-get install -qy python3-sphinx texlive-xetex texlive-latex-extra + +# Core build utilities +RUN apt-get install -qy libgmp-dev:i386 + +# Get i386 GHC bindist for 32 bit CI builds. +RUN cd /tmp && curl https://downloads.haskell.org/~ghc/8.4.2/ghc-8.4.2-i386-deb8-linux.tar.xz | tar -Jx +RUN cd /tmp/ghc-8.4.2 && setarch i386 ./configure --prefix=/opt/ghc-i386/8.4.2 CFLAGS=-m32 --target=i386-unknown-linux --build=i386-unknown-linux --host=i386-unknown-linux +RUN cd /tmp/ghc-8.4.2 && make install +RUN rm -rf /tmp/ghc-8.4.2 +ENV PATH /opt/ghc-i386/8.4.2/bin:$PATH + +# Get Cabal +RUN cd /tmp && \ + curl https://www.haskell.org/cabal/release/cabal-install-2.2.0.0/cabal-install-2.2.0.0-i386-unknown-linux.tar.gz | tar -zx && \ + mv cabal /usr/local/bin/cabal + +# Create a normal user. +RUN adduser ghc --gecos "GHC builds" --disabled-password +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc + +# Build Haskell tools +RUN cabal update && \ + cabal install hscolour happy alex +ENV PATH /home/ghc/.cabal/bin:$PATH + +WORKDIR /home/ghc/ + +CMD ["bash"] diff --git a/.circleci/images/linters/Dockerfile b/.circleci/images/linters/Dockerfile new file mode 100644 index 0000000000..cd9aa30e69 --- /dev/null +++ b/.circleci/images/linters/Dockerfile @@ -0,0 +1,30 @@ +FROM debian:stretch + +ENV LANG C.UTF-8 + +RUN apt-get update -qq; apt-get install -qy gnupg sudo git python3 + +RUN echo 'deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main' > /etc/apt/sources.list.d/ghc.list +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6F88286 +RUN apt-get update -qq + +# Basic Haskell toolchain +RUN apt-get install -qy cabal-install-2.2 ghc-8.4.2 +ENV PATH /home/ghc/.local/bin:/opt/cabal/2.2/bin:/opt/ghc/8.4.2/bin:$PATH + +# Create a normal user. +RUN adduser ghc --gecos "GHC builds" --disabled-password +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc +WORKDIR /home/ghc/ + +# Build Linting tools +RUN cabal update + +RUN git clone git://github.com/haskell-infra/git-haskell-org-hooks && \ + cd git-haskell-org-hooks && \ + cabal install + +ENV PATH /home/ghc/.cabal/bin:$PATH + +CMD ["bash"] diff --git a/.circleci/images/powerpc64le-linux-deb9-cross/Dockerfile b/.circleci/images/powerpc64le-linux-deb9-cross/Dockerfile new file mode 100644 index 0000000000..bf0bbf3ffd --- /dev/null +++ b/.circleci/images/powerpc64le-linux-deb9-cross/Dockerfile @@ -0,0 +1,7 @@ +FROM ghcci/x86_64-linux-deb9 + +ENV LANG C.UTF-8 +RUN apt-get install qemu-user && \ + dpkg --add-architecture ppc64el && \ + apt-get install crossbuild-essential-ppc64el libc6:ppc64el ncurses:ppc64el + diff --git a/.circleci/images/update-image b/.circleci/images/update-image index 5a3cae506c..dd69122c1f 100755 --- a/.circleci/images/update-image +++ b/.circleci/images/update-image @@ -17,17 +17,18 @@ fi name=$1 version=$2 -versions="$(curl -s https://registry.hub.docker.com/v1/repositories/$repo/$name/tags | jq -r .[].name)" -if echo "$versions" | grep $version > /dev/null; then - echo "Version $version of $name already exists" - echo "Previous versions are:" - echo "$versions" - exit 1 -fi +#versions="$(curl -s https://registry.hub.docker.com/v1/repositories/$repo/$name/tags | jq -r .[].name)" +#if echo "$versions" | grep $version > /dev/null; then +# echo "Version $version of $name already exists" +# echo "Previous versions are:" +# echo "$versions" +# exit 1 +#fi docker build $name -t $repo/$name:$version docker push $repo/$name:$version repo_name="$repo/$name" sed -i -E -e "s%$repo_name"':[0-9]+(\.[0-9]+)*%'"$repo_name:$version%" ../config.yml +sed -i -E -e "s%$repo_name"':[0-9]+(\.[0-9]+)*%'"$repo_name:$version%" ../../.gitlab-ci.yml echo "Built, pushed, and bumped $name:$version" diff --git a/.circleci/images/x86_64-freebsd/Dockerfile b/.circleci/images/x86_64-freebsd/Dockerfile index 9c51cc021c..1098d7a9ac 100644 --- a/.circleci/images/x86_64-freebsd/Dockerfile +++ b/.circleci/images/x86_64-freebsd/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ software-properties-common \ sudo \ wget \ + jq \ xz-utils COPY build-toolchain.sh /tmp/ diff --git a/.circleci/images/x86_64-linux-centos7/Dockerfile b/.circleci/images/x86_64-linux-centos7/Dockerfile new file mode 100644 index 0000000000..32acea5ca3 --- /dev/null +++ b/.circleci/images/x86_64-linux-centos7/Dockerfile @@ -0,0 +1,54 @@ +FROM centos:7 + +ENV LANG C.UTF-8 + +# Core build utilities +RUN yum -y install coreutils binutils which git make \ + automake autoconf gcc perl python3 texinfo xz lbzip2 \ + patch openssh-clients sudo zlib-devel sqlite \ + ncurses-compat-libs gmp-devel ncurses-devel gcc-c++ findutils \ + curl wget jq + +# Python 3 +RUN yum install -y https://centos7.iuscommunity.org/ius-release.rpm; \ + yum install -y python34u python34u-libs + +# Documentation tools +RUN yum -y install python-sphinx \ + texlive texlive-latex texlive-xetex \ + texlive-collection-latex texlive-collection-latexrecommended \ + texlive-xetex-def texlive-collection-xetex \ + python-sphinx-latex dejavu-sans-fonts dejavu-serif-fonts \ + dejavu-sans-mono-fonts + +# This is in the PATH when I ssh into the CircleCI machine but somehow +# sphinx-build isn't found during configure unless we explicitly +# add it here as well; perhaps PATH is being overridden by CircleCI's +# infrastructure? +ENV PATH /usr/libexec/python3-sphinx:$PATH + +# systemd isn't running so remove it from nsswitch.conf +# Failing to do this will result in testsuite failures due to +# non-functional user lookup (#15230). +RUN sed -i -e 's/systemd//g' /etc/nsswitch.conf + +# Install GHC and cabal +RUN cd /tmp && curl https://downloads.haskell.org/~ghc/8.4.2/ghc-8.4.2-x86_64-deb8-linux.tar.xz | tar -Jx +RUN cd /tmp/ghc-8.4.2 && ./configure --prefix=/opt/ghc/8.4.2 +RUN cd /tmp/ghc-8.4.2 && make install +RUN mkdir -p /opt/cabal/bin +RUN cd /opt/cabal/bin && curl https://www.haskell.org/cabal/release/cabal-install-2.2.0.0/cabal-install-2.2.0.0-x86_64-unknown-linux.tar.gz | tar -zx +ENV PATH /opt/ghc/8.4.2/bin:/opt/cabal/bin:$PATH + +# Create a normal user. +RUN adduser ghc --comment "GHC builds" +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc +WORKDIR /home/ghc/ + +# Install Alex, Happy, and HsColor with Cabal +RUN cabal update && \ + cabal install hscolour happy alex +ENV PATH /home/ghc/.cabal/bin:$PATH + +CMD ["bash"] diff --git a/.circleci/images/x86_64-linux-deb8/Dockerfile b/.circleci/images/x86_64-linux-deb8/Dockerfile new file mode 100644 index 0000000000..0491ad9bba --- /dev/null +++ b/.circleci/images/x86_64-linux-deb8/Dockerfile @@ -0,0 +1,35 @@ +FROM debian:jessie + +ENV LANG C.UTF-8 + +RUN echo 'deb http://ppa.launchpad.net/hvr/ghc/ubuntu trusty main' > /etc/apt/sources.list.d/ghc.list +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6F88286 +RUN apt-get update -qq + +# Core build utilities +RUN apt-get install -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \ + ca-certificates g++ git make automake autoconf gcc \ + perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo \ + jq wget curl + +# Documentation tools +RUN apt-get install -qy python3-sphinx texlive-xetex texlive-latex-extra + +# Basic Haskell toolchain +RUN apt-get install -qy cabal-install-2.2 ghc-8.4.2 + +ENV PATH /home/ghc/.local/bin:/opt/cabal/2.2/bin:/opt/ghc/8.4.2/bin:$PATH + +# Create a normal user. +RUN adduser ghc --gecos "GHC builds" --disabled-password +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc + +# Build Haskell tools +RUN cabal update && \ + cabal install hscolour happy alex +ENV PATH /home/ghc/.cabal/bin:$PATH + +WORKDIR /home/ghc/ + +CMD ["bash"] diff --git a/.circleci/images/x86_64-linux-deb9/Dockerfile b/.circleci/images/x86_64-linux-deb9/Dockerfile new file mode 100644 index 0000000000..5341abe6ef --- /dev/null +++ b/.circleci/images/x86_64-linux-deb9/Dockerfile @@ -0,0 +1,46 @@ +FROM debian:stretch + +ENV LANG C.UTF-8 + +RUN apt-get update -qq; apt-get install -qy gnupg + +RUN echo 'deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main' > /etc/apt/sources.list.d/ghc.list +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6F88286 +RUN apt-get update -qq + +# Core build utilities +RUN apt-get install -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \ + ca-certificates g++ git make automake autoconf gcc \ + perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo \ + jq wget curl + +# Documentation tools +RUN apt-get install -qy python3-sphinx texlive-xetex texlive-latex-extra + +# Basic Haskell toolchain +RUN apt-get install -qy cabal-install-2.2 ghc-8.4.2 + +ENV PATH /home/ghc/.local/bin:/opt/cabal/2.2/bin:/opt/ghc/8.4.2/bin:$PATH + +# LLVM +ENV LLVM_TARBALL http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz +ENV PATH /opt/llvm/bin:$PATH +RUN curl $LLVM_TARBALL | tar -xJC ..; \ + mkdir /opt/llvm; \ + cp -R clang+llvm*/* /opt/llvm; \ + rm -R clang+llvm*; \ + llc --version + +# Create a normal user. +RUN adduser ghc --gecos "GHC builds" --disabled-password +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc + +# Build Haskell tools +RUN cabal update && \ + cabal install hscolour happy alex +ENV PATH /home/ghc/.cabal/bin:$PATH + +WORKDIR /home/ghc/ + +CMD ["bash"] diff --git a/.circleci/images/x86_64-linux-fedora27/Dockerfile b/.circleci/images/x86_64-linux-fedora27/Dockerfile new file mode 100644 index 0000000000..6fad77f1c5 --- /dev/null +++ b/.circleci/images/x86_64-linux-fedora27/Dockerfile @@ -0,0 +1,50 @@ +FROM fedora:27 + +ENV LANG C.UTF-8 + +# Core build utilities +RUN dnf -y install coreutils binutils which git make \ + automake autoconf gcc perl python3 texinfo xz lbzip2 \ + patch openssh-clients sudo zlib-devel sqlite \ + ncurses-compat-libs gmp-devel ncurses-devel gcc-c++ findutils \ + curl wget jq + +# Documentation tools +RUN dnf -y install python3-sphinx \ + texlive texlive-latex texlive-xetex \ + texlive-collection-latex texlive-collection-latexrecommended \ + texlive-xetex-def texlive-collection-xetex \ + python-sphinx-latex dejavu-sans-fonts dejavu-serif-fonts \ + dejavu-sans-mono-fonts + +# This is in the PATH when I ssh into the CircleCI machine but somehow +# sphinx-build isn't found during configure unless we explicitly +# add it here as well; perhaps PATH is being overridden by CircleCI's +# infrastructure? +ENV PATH /usr/libexec/python3-sphinx:$PATH + +# systemd isn't running so remove it from nsswitch.conf +# Failing to do this will result in testsuite failures due to +# non-functional user lookup (#15230). +RUN sed -i -e 's/systemd//g' /etc/nsswitch.conf + +# Install GHC and cabal +RUN cd /tmp && curl https://downloads.haskell.org/~ghc/8.4.2/ghc-8.4.2-x86_64-deb8-linux.tar.xz | tar -Jx +RUN cd /tmp/ghc-8.4.2 && ./configure --prefix=/opt/ghc/8.4.2 +RUN cd /tmp/ghc-8.4.2 && make install +RUN mkdir -p /opt/cabal/bin +RUN cd /opt/cabal/bin && curl https://www.haskell.org/cabal/release/cabal-install-2.2.0.0/cabal-install-2.2.0.0-x86_64-unknown-linux.tar.gz | tar -zx +ENV PATH /opt/ghc/8.4.2/bin:/opt/cabal/bin:$PATH + +# Create a normal user. +RUN adduser ghc --comment "GHC builds" +RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc +USER ghc +WORKDIR /home/ghc/ + +# Install Alex, Happy, and HsColor with Cabal +RUN cabal update && \ + cabal install hscolour happy alex +ENV PATH /home/ghc/.cabal/bin:$PATH + +CMD ["bash"] diff --git a/.circleci/prepare-system.sh b/.circleci/prepare-system.sh index 7d8cac60a2..804e0fd890 100755 --- a/.circleci/prepare-system.sh +++ b/.circleci/prepare-system.sh @@ -11,6 +11,8 @@ hackage_index_state="@1522046735" if [[ -z ${BUILD_SPHINX_HTML:-} ]]; then BUILD_SPHINX_HTML=YES; fi if [[ -z ${BUILD_SPHINX_PDF:-} ]]; then BUILD_SPHINX_PDF=YES; fi +if [[ -z ${INTEGER_LIBRARY:-} ]]; then INTEGER_LIBRARY=integer-gmp; fi +if [[ -z ${BUILD_FLAVOUR:-} ]]; then BUILD_FLAVOUR=perf; fi cat > mk/build.mk <<EOF V=1 @@ -20,6 +22,14 @@ HSCOLOUR_SRCS=YES BUILD_SPHINX_HTML=$BUILD_SPHINX_HTML BUILD_SPHINX_PDF=$BUILD_SPHINX_PDF BeConservative=YES +INTEGER_LIBRARY=$INTEGER_LIBRARY +EOF + +cat <<EOF >> mk/build.mk +BuildFlavour=$BUILD_FLAVOUR +ifneq "\$(BuildFlavour)" "" +include mk/flavours/\$(BuildFlavour).mk +endif EOF case "$(uname)" in @@ -27,15 +37,6 @@ case "$(uname)" in if [[ -n ${TARGET:-} ]]; then if [[ $TARGET = FreeBSD ]]; then # cross-compiling to FreeBSD - add-apt-repository -y ppa:hvr/ghc - apt-get update -qq - apt-get install -qy ghc-8.0.2 cabal-install-1.24 alex happy \ - ncurses-dev git make automake autoconf gcc perl \ - python3 texinfo xz-utils lbzip2 patch - cabal update - cabal install --reinstall hscolour --index-state=$hackage_index_state - ln -s $HOME/.cabal/bin/HsColour /usr/local/bin/HsColour - echo 'HADDOCK_DOCS = NO' >> mk/build.mk echo 'WERROR=' >> mk/build.mk # https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables @@ -43,10 +44,6 @@ case "$(uname)" in else fail "TARGET=$target not supported" fi - else - cabal update -v - cabal install --reinstall hscolour - sudo ln -s /home/ghc/.cabal/bin/HsColour /usr/local/bin/HsColour || true fi ;; diff --git a/.circleci/push-test-metrics.sh b/.circleci/push-test-metrics.sh new file mode 100755 index 0000000000..e383a4c4e7 --- /dev/null +++ b/.circleci/push-test-metrics.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# vim: sw=2 et +set -euo pipefail + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:ghc/ghc.git" ]; then + exit 0 +fi + +GHC_ORIGIN=git@git.haskell.org:ghc + +# Add git.haskell.org as a known host. +echo "|1|F3mPVCE55+KfApNIMYQ3Dv39sGE=|1bRkvJEJhAN2R0LE/lAjFCEJGl0= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBUZS9jGBkE5UzpSo6irnIgcQcfzvbuIOsFc8+N61FwtZncRntbaKPuUimOFPgeaUZLl6Iajz6IIs7aduU0/v+I=" >> ~/.ssh/known_hosts +echo "|1|2VUMjYSRVpT2qJPA0rA9ap9xILY=|5OThkI4ED9V0J+Es7D5FOD55Klk= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+3TLluLAO4lkW60W+N2DFkS+WoRFGqLwHzgd1ifxG9TIm31wChPY3E/hgMnJmgGqWCF4UDUemmyCycEaL7FtKfzjTAclg9EfpQnozyE3T5hIo2WL7SN5O8ttG/bYGuDnn14jLnWwJyN4oz/znWFiDG9e2Oc9YFNlQ+PK8ae5xR4gqBB7EOoj9J1EiPqG2OXRr5Mei3TLsRDU6fnz/e4oFJpKWWeN6M63oePv0qoaGjxcrATZUWsuWrxVMmYo9kP1xRuFJbAUw2m4uVP+793SW1zxySi1HBMtJG+gCDdZZSwYbkV1hassLWBHv1qPttncfX8Zek3Z3VolaTmfWJTo9" >> ~/.ssh/known_hosts + +# Check that a git notes dont already exist. +# This is a percausion as we reset refs/notes/perf and we want to avoid data loss. +if [ $(git notes --ref=perf list | wc -l) -ne 0 ] +then + fail "Found an existing git note on HEAD. Expected no git note." +fi + +# Assert that the METRICS_FILE exists and can be read. +if [ "$METRICS_FILE" = "" ] || ! [ -r $METRICS_FILE ] +then + fail "Metrics file not found: $METRICS_FILE" +fi + +# Reset the git notes and append the metrics file to the notes, then push and return the result. +# This is favoured over a git notes merge as it avoids potential data loss/duplication from the merge strategy. +function reset_append_note_push { + git fetch -f $GHC_ORIGIN refs/notes/perf:refs/notes/perf || true + echo "git notes --ref=perf append -F $METRICS_FILE HEAD" + git notes --ref=perf append -F $METRICS_FILE HEAD + git push $GHC_ORIGIN refs/notes/perf +} + +# Push the metrics file as a git note. This may fail if another task pushes a note first. In that case +# the latest note is fetched and appended. +MAX_RETRY=20 +until reset_append_note_push || [ $MAX_RETRY -le 0 ] +do + ((MAX_RETRY--)) + echo "" + echo "Failed to push git notes. Fetching, appending, and retrying... $MAX_RETRY retries left." +done diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..06d7589bca --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,401 @@ +variables: + GIT_SSL_NO_VERIFY: "1" + +before_script: + - python3 .gitlab/fix-submodules.py + - git submodule sync --recursive + - git submodule update --init --recursive + - git checkout .gitmodules + +stages: + - lint + - build + - full-build + +############################################################ +# Runner Tags +############################################################ +# +# * x86_64-linux: Any Docker-capable x86_64 Linux machine +# * aarch64-linux: Any Docker-capable AArch64 Linux machine +# * x86_64-windows: A x86_64 Windows machine +# * lint: Any Docker-capable x86_64 Linux machine; distinct from +# x86_64-linux to ensure low-latency availability. +# + + +############################################################ +# Linting +############################################################ + +ghc-linters: + stage: lint + image: ghcci/linters:0.1 + script: + - | + if [ -n "$CI_MERGE_REQUEST_ID" ]; then + base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)" + validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA) + submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA) + validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA) + .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA + .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA + fi + tags: + - lint + +############################################################ +# Validation via Pipelines (hadrian) +############################################################ + +.validate-hadrian: + allow_failure: true + script: + - git clean -xdf && git submodule foreach git clean -xdf + - bash .circleci/prepare-system.sh + - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi + - ./boot + - ./configure $CONFIGURE_ARGS + - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` + cache: + key: hadrian + paths: + - cabal-cache + +validate-x86_64-linux-deb8-hadrian: + extends: .validate-hadrian + stage: build + image: ghcci/x86_64-linux-deb8:0.1 + before_script: + # workaround for docker permissions + - sudo chown ghc:ghc -R . + - python3 .gitlab/fix-submodules.py + - git submodule sync --recursive + - git submodule update --init --recursive + - git checkout .gitmodules + tags: + - x86_64-linux + +############################################################ +# Validation via Pipelines (make) +############################################################ + +.validate: + variables: + TEST_TYPE: test + before_script: + - git clean -xdf && git submodule foreach git clean -xdf + script: + - ./boot + - ./configure $CONFIGURE_ARGS + - | + THREADS=`mk/detect-cpu-count.sh` + make V=0 -j$THREADS + - | + make binary-dist TAR_COMP_OPTS="-1" + mv ghc-*.tar.xz ghc.tar.xz + - | + THREADS=`mk/detect-cpu-count.sh` + make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml + artifacts: + reports: + junit: junit.xml + expire_in: 2 week + paths: + - ghc.tar.xz + - junit.xml + +validate-x86_64-darwin: + extends: .validate + stage: full-build + allow_failure: true + tags: + - x86_64-darwin + variables: + GHC_VERSION: 8.6.3 + MACOSX_DEPLOYMENT_TARGET: "10.7" + # Only Sierra and onwards supports clock_gettime. See #12858 + ac_cv_func_clock_gettime: "no" + LANG: "en_US.UTF-8" + before_script: + - git clean -xdf && git submodule foreach git clean -xdf + - python .gitlab/fix-submodules.py + - git submodule sync --recursive + - git submodule update --init --recursive + - git checkout .gitmodules + + - bash .gitlab/darwin-init.sh + - PATH="`pwd`/toolchain/bin:$PATH" + # Disable sphinx PDF output as MacTeX apparently doesn't provide xelatex + - echo "BUILD_SPHINX_PDF=NO" >> mk/build.mk + - echo "libraries/integer-gmp_CONFIGURE_OPTS += --configure-option=--with-intree-gmp" >> mk/build.mk + after_script: + - cp -Rf $HOME/.cabal cabal-cache + artifacts: + when: always + expire_in: 2 week + cache: + key: darwin + paths: + - cabal-cache + - toolchain + +.validate-linux: + extends: .validate + tags: + - x86_64-linux + before_script: + - git clean -xdf && git submodule foreach git clean -xdf + - python3 .gitlab/fix-submodules.py + - git submodule sync --recursive + - git submodule update --init --recursive + - git checkout .gitmodules + + - bash .circleci/prepare-system.sh + # workaround for docker permissions + - sudo chown ghc:ghc -R . + after_script: + - cp -Rf $HOME/.cabal cabal-cache + cache: + key: linux + paths: + - cabal-cache + - toolchain + +validate-aarch64-linux-deb9: + extends: .validate-linux + stage: full-build + image: ghcci/aarch64-linux-deb9:0.1 + allow_failure: true + artifacts: + when: always + expire_in: 2 week + cache: + key: linux-aarch64-deb9 + tags: + - aarch64-linux + +nightly-aarch64-linux-deb9: + extends: validate-aarch64-linux-deb9 + stage: full-build + artifacts: + expire_in: 2 year + variables: + TEST_TYPE: slowtest + only: + variables: + - $NIGHTLY + +validate-i386-linux-deb9: + extends: .validate-linux + stage: full-build + image: ghcci/i386-linux-deb9:0.1 + allow_failure: true + artifacts: + when: always + expire_in: 2 week + cache: + key: linux-i386-deb9 + +nightly-i386-linux-deb9: + extends: .validate-linux + stage: full-build + image: ghcci/i386-linux-deb9:0.1 + allow_failure: true + variables: + TEST_TYPE: slowtest + artifacts: + when: always + expire_in: 2 week + only: + variables: + - $NIGHTLY + cache: + key: linux-i386-deb9 + +validate-x86_64-linux-deb9: + extends: .validate-linux + stage: build + image: ghcci/x86_64-linux-deb9:0.2 + artifacts: + when: always + expire_in: 2 week + cache: + key: linux-x86_64-deb9 + +nightly-x86_64-linux-deb9: + extends: validate-x86_64-linux-deb9 + stage: build + artifacts: + expire_in: 2 year + variables: + TEST_TYPE: slowtest + only: + variables: + - $NIGHTLY + +validate-x86_64-linux-deb9-llvm: + extends: .validate-linux + stage: full-build + allow_failure: true + image: ghcci/x86_64-linux-deb9:0.2 + variables: + BUILD_FLAVOUR: perf-llvm + cache: + key: linux-x86_64-deb9 + +validate-x86_64-linux-deb8: + extends: .validate-linux + stage: full-build + image: ghcci/x86_64-linux-deb8:0.1 + cache: + key: linux-x86_64-deb8 + artifacts: + when: always + expire_in: 2 week + +validate-x86_64-linux-fedora27: + extends: .validate-linux + stage: full-build + image: ghcci/x86_64-linux-fedora27:0.1 + cache: + key: linux-x86_64-fedora27 + artifacts: + when: always + expire_in: 2 week + +validate-x86_64-linux-deb9-integer-simple: + extends: .validate-linux + stage: full-build + variables: + INTEGER_LIBRARY: integer-simple + image: ghcci/x86_64-linux-deb9:0.2 + cache: + key: linux-x86_64-deb9 + +nightly-x86_64-linux-deb9-integer-simple: + extends: validate-x86_64-linux-deb9-integer-simple + stage: full-build + artifacts: + expire_in: 2 year + variables: + TEST_TYPE: slowtest + only: + variables: + - $NIGHTLY + +validate-x86_64-linux-deb9-unreg: + extends: .validate-linux + stage: full-build + variables: + CONFIGURE_ARGS: --enable-unregisterised + image: ghcci/x86_64-linux-deb9:0.2 + cache: + key: linux-x86_64-deb9 + +############################################################ +# Validation via Pipelines (Windows) +############################################################ + +.validate-windows: + before_script: + - git clean -xdf + - git submodule foreach git clean -xdf + + # Use a local temporary directory to ensure that concurrent builds don't + # interfere with one another + - | + mkdir tmp + set TMP=%cd%\tmp + set TEMP=%cd%\tmp + + - set PATH=C:\msys64\usr\bin;%PATH% + - python .gitlab/fix-submodules.py + - git submodule sync --recursive + - git submodule update --init --recursive + - git checkout .gitmodules + - bash .gitlab/win32-init.sh + after_script: + - rd /s /q tmp + - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache + - bash -c 'make clean || true' + cache: + paths: + - cabal-cache + - ghc-8.6.2 + - ghc-tarballs + +validate-x86_64-windows-hadrian: + extends: .validate-windows + stage: full-build + # due to #16073 + allow_failure: true + variables: + GHC_VERSION: "8.6.2" + LANG: "en_US.UTF-8" + script: + - | + set MSYSTEM=MINGW64 + python boot + bash -c './configure --with-ghc=`pwd`/toolchain/bin/ghc --enable-tarballs-autodownload HappyCmd=`pwd`/toolchain/bin/happy AlexCmd=`pwd`/toolchain/bin/alex' + mkdir -p _build + cp -R inplace/mingw _build/mingw + # FIXME: --no-lint due to #15950 + - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --no-lint" + - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist" + - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml' + cache: + key: x86_64-windows + tags: + - x86_64-windows + +validate-x86_64-windows: + extends: .validate-windows + stage: build + # due to #16084 + allow_failure: true + variables: + GHC_VERSION: "8.6.2" + LANG: "en_US.UTF-8" + script: + - | + set MSYSTEM=MINGW64 + python boot + bash -c './configure --with-ghc=`pwd`/toolchain/bin/ghc --enable-tarballs-autodownload HappyCmd=`pwd`/toolchain/bin/happy AlexCmd=`pwd`/toolchain/bin/alex' + - bash -c "echo include mk/flavours/quick.mk > mk/build.mk" + - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`" + - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml' + cache: + key: x86_64-windows + tags: + - x86_64-windows + +############################################################ +# Validation via CircleCI +############################################################ + +.circleci: + stage: build + image: ghcci/x86_64-linux-deb8:0.1 + artifacts: + when: always + expire_in: 2 week + reports: + junit: junit.xml + paths: + - ghc.tar.xz + - junit.xml + tags: + - circleci + +# All validation jobs keep the bindists and test results are artifacts, +# when we get far enough to generate them. +# +# This requires updating the maximum artifacts size limit in Gitlab to +# something like 200MB. + +.circleci-validate-x86_64-darwin: + extends: .circleci + stage: full-build + allow_failure: true + script: ".gitlab/circle-ci-job.sh validate-x86_64-darwin" diff --git a/.gitlab/circle-ci-job.sh b/.gitlab/circle-ci-job.sh new file mode 100755 index 0000000000..bbb00e062a --- /dev/null +++ b/.gitlab/circle-ci-job.sh @@ -0,0 +1,110 @@ +# Circle CI "backend" for Gitlab CI +# ================================= +# +# Usage example: +# .gitlab/circle-ci-job.sh validate-x86_64-linux +# +# There are two things to configure to get artifacts to be +# uploaded to gitlab properly: +# +# - At https://<gitlab host>/admin/application_settings, expand the +# Continuous Integration and Deployment section and set the +# "Maximum artifacts size (MB)" field to something large enough +# to contain the bindists (the test reports are tiny in comparison). +# 500MB seems to work fine, but 200MB might be sufficient. +# +# - If gitlab is exposed behind some form of proxy (e.g nginx), make sure +# the maximum client request body size is large enough to contain all the +# artifacts of a build. For nginx, this would be the following configuration +# option: https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size +# (which can be set with services.nginx.clientMaxBodySize on nixos). + +#!/usr/bin/env sh + +set -e + +GHCCI_URL="localhost:8888" + +[ $# -gt 0 ] || (echo You need to pass the Circle CI job type as argument to this script; exit 1) +[ ${CI_RUNNER_ID:-} ] || (echo "CI_RUNNER_ID is not set"; exit 1) +[ ${CI_JOB_ID:-} ] || (echo "CI_JOB_ID is not set"; exit 1) +[ ${CI_COMMIT_SHA:-} ] || (echo "CI_COMMIT_SHA is not set"; exit 1) +[ ${CI_REPOSITORY_URL:-} ] || (echo "CI_REPOSITORY_URL is not set"; exit 1) +[ ${CI_PIPELINE_ID:-} ] || (echo "CI_PIPELINE_ID is not set"; exit 1) +# the first argument to this script is the Circle CI job type: +# validate-x86_64-linux, validate-i386-linux, ... +CIRCLE_JOB="circleci-$1" + +gitlab_user=$(echo $CI_REPOSITORY_URL | cut -d/ -f4) +gitlab_repo=$(echo $CI_REPOSITORY_URL | cut -d/ -f5 | cut -d. -f1) + +BODY="{ \"jobType\": \"$CIRCLE_JOB\", \"source\": { \"user\": \"$gitlab_user\", \"project\":\"$gitlab_repo\", \"commit\":\"$CI_COMMIT_SHA\" }, \"pipelineID\": $CI_PIPELINE_ID, \"runnerID\": $CI_RUNNER_ID, \"jobID\": $CI_JOB_ID }" + + +RESP=$(curl -s -XPOST -H "Content-Type: application/json" -d "$BODY" \ + http://${GHCCI_URL}/job) + +if [ $? -eq 0 ]; then + build_num=$(echo $RESP | jq '.build_num') + circle_url=$(echo $RESP | jq '.url') +else + echo "Couldn't submit job" + echo $RESP + exit 1 +fi + +echo Circle CI build number: $build_num +echo Circle CI build page: $circle_url + +outcome="null" +STATUS_URL="http://${GHCCI_URL}/job/${build_num}" +STATUS_RESP="" + +while [ "$outcome" == "null" ]; do + sleep 30s + STATUS_RESP=$(curl -s $STATUS_URL) + if [ $? -eq 0 ]; then + new_outcome=$(echo $STATUS_RESP | jq '.outcome') + jq_exitcode=$? + if [ "$new_outcome" == "null" ] && [ $jq_exitcode -ne 0 ]; then + echo "Couldn't read 'outcome' field in JSON:" + echo $STATUS_RESP + echo "Skipping" + else + outcome="$new_outcome" + fi + else + echo "curl failed:" + echo $STATUS_RESP + echo "Skipping" + fi +done + +if [ "$outcome" == "\"success\"" ]; then + echo The build passed + artifactsBody=$(curl -s http://${GHCCI_URL}/job/${build_num}/artifacts) + (echo $artifactsBody | jq '.[] | .url' | xargs wget -q) || echo "No artifacts" + exit 0 +else + echo The build failed + + artifactsBody=$(curl -s http://${GHCCI_URL}/job/${build_num}/artifacts) + (echo $artifactsBody | jq '.[] | .url' | xargs wget -q) || echo "No artifacts" + + failing_step=$(echo $STATUS_RESP | jq '.steps | .[] | .actions | .[] | select(.status != "success")') + failing_step_name=$(echo $failing_step | jq '.name' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/') + echo "Failing step: $failing_step_name" + + failing_cmds=$(echo $failing_step | jq '.bash_command' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/') + echo "Failing command(s):" + echo $failing_cmds + + log_url=$(echo $failing_step | jq '.output_url' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/') + echo "Log url: $log_url" + + last_log_lines=$(curl -s $log_url | gunzip | jq '.[] | select(.type == "out") | .message' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/' | tail -50) + echo End of the build log: + echo $last_log_lines + + exit 1 +fi diff --git a/.gitlab/darwin-init.sh b/.gitlab/darwin-init.sh new file mode 100644 index 0000000000..ba5de9db35 --- /dev/null +++ b/.gitlab/darwin-init.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +toolchain=`pwd`/toolchain +PATH="$toolchain/bin:$PATH" + +if [ -d "`pwd`/cabal-cache" ]; then + cp -Rf cabal-cache $HOME/.cabal +fi + +if [ ! -e $toolchain/bin/ghc ]; then + mkdir -p tmp + cd tmp + ghc_tarball="https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-x86_64-apple-darwin.tar.xz" + echo "Fetching GHC from $ghc_tarball" + curl $ghc_tarball | tar -xJ + cd ghc-$GHC_VERSION + ./configure --prefix=$toolchain + make install + cd ../.. + rm -Rf tmp +fi + +if [ ! -e $toolchain/bin/cabal ]; then + cabal_tarball="https://downloads.haskell.org/~cabal/cabal-install-latest/cabal-install-2.4.1.0-x86_64-apple-darwin-sierra.tar.xz" + echo "Fetching cabal-install from $cabal_tarball" + curl $cabal_tarball | tar -xz + mv cabal $toolchain/bin +fi + +if [ ! -e $toolchain/bin/happy ]; then + cabal update + cabal new-install happy --symlink-bindir=$toolchain/bin +fi + +if [ ! -e $toolchain/bin/alex ]; then + cabal update + cabal new-install alex --symlink-bindir=$toolchain/bin +fi + diff --git a/.gitlab/fix-submodules.py b/.gitlab/fix-submodules.py new file mode 100644 index 0000000000..2ff8e41974 --- /dev/null +++ b/.gitlab/fix-submodules.py @@ -0,0 +1,8 @@ +#!/usr/bin/python + +import re + +x = open('.gitmodules').read() +x = re.sub(r"url *= *\.\.", "url = https://gitlab.haskell.org/ghc", x) +open('.gitmodules', 'w').write(x) + diff --git a/.gitlab/linters/check-cpp.py b/.gitlab/linters/check-cpp.py new file mode 100755 index 0000000000..c89451399e --- /dev/null +++ b/.gitlab/linters/check-cpp.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# A linter to warn for ASSERT macros which are separated from their argument +# list by a space, which Clang's CPP barfs on + +from linter import run_linters, RegexpLinter + +linters = [ + RegexpLinter(r'WARN\s+\(', + message='CPP macros should not have a space between the macro name and their argument list'), + RegexpLinter(r'ASSERT\s+\(', + message='CPP macros should not have a space between the macro name and their argument list'), + RegexpLinter(r'ASSERT2\s+\(', + message='CPP macros should not have a space between the macro name and their argument list'), + RegexpLinter(r'#ifdef\s+', + message='`#if defined(x)` is preferred to `#ifdef x`'), + RegexpLinter(r'#if\s+defined\s+', + message='`#if defined(x)` is preferred to `#if defined x`'), + RegexpLinter(r'#ifndef\s+', + message='`#if !defined(x)` is preferred to `#ifndef x`'), +] + +if __name__ == '__main__': + run_linters(linters) diff --git a/.gitlab/linters/check-makefiles.py b/.gitlab/linters/check-makefiles.py new file mode 100755 index 0000000000..c97838beb5 --- /dev/null +++ b/.gitlab/linters/check-makefiles.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +""" +Warn for use of `--interactive` inside Makefiles (#11468). + +Encourage the use of `$(TEST_HC_OPTS_INTERACTIVE)` instead of +`$(TEST_HC_OPTS) --interactive -ignore-dot-ghci -v0`. It's too easy to +forget one of those flags when adding a new test. +""" + +from linter import run_linters, RegexpLinter + +linters = [ + RegexpLinter(r'--interactive', + message = "Warning: Use `$(TEST_HC_OPTS_INTERACTIVE)` instead of `--interactive -ignore-dot-ghci -v0`.") +] + +if __name__ == '__main__': + run_linters(linters) #$, subdir='testsuite') diff --git a/.gitlab/linters/linter.py b/.gitlab/linters/linter.py new file mode 100644 index 0000000000..ec4f3581a8 --- /dev/null +++ b/.gitlab/linters/linter.py @@ -0,0 +1,109 @@ +""" +Utilities for linters +""" + +import os +import sys +import re +import textwrap +import subprocess +from typing import List, Optional +from collections import namedtuple + +def lint_failure(file, line_no, line_content, message): + """ Print a lint failure message. """ + wrapper = textwrap.TextWrapper(initial_indent=' ', + subsequent_indent=' ') + body = wrapper.fill(message) + msg = ''' + {file}: + + | + {line_no:5d} | {line_content} + | + + {body} + '''.format(file=file, line_no=line_no, + line_content=line_content, + body=body) + + print(textwrap.dedent(msg)) + +def get_changed_files(base_commit, head_commit, + subdir: str = '.'): + """ Get the files changed by the given range of commits. """ + cmd = ['git', 'diff', '--name-only', + base_commit, head_commit, '--', subdir] + files = subprocess.check_output(cmd) + return files.decode('UTF-8').split('\n') + +Warning = namedtuple('Warning', 'path,line_no,line_content,message') + +class Linter(object): + """ + A :class:`Linter` must implement :func:`lint`, which looks at the + given path and calls :func:`add_warning` for any lint issues found. + """ + def __init__(self): + self.warnings = [] # type: List[Warning] + + def add_warning(self, w: Warning): + self.warnings.append(w) + + def lint(self, path): + pass + +class LineLinter(Linter): + """ + A :class:`LineLinter` must implement :func:`lint_line`, which looks at + the given line from a file and calls :func:`add_warning` for any lint + issues found. + """ + def lint(self, path): + if os.path.isfile(path): + with open(path, 'r') as f: + for line_no, line in enumerate(f): + self.lint_line(path, line_no+1, line) + + def lint_line(self, path, line_no, line): + pass + +class RegexpLinter(LineLinter): + """ + A :class:`RegexpLinter` produces the given warning message for + all lines matching the given regular expression. + """ + def __init__(self, regex, message): + LineLinter.__init__(self) + self.re = re.compile(regex) + self.message = message + + def lint_line(self, path, line_no, line): + if self.re.search(line): + w = Warning(path=path, line_no=line_no, line_content=line[:-1], + message=self.message) + self.add_warning(w) + +def run_linters(linters: List[Linter], + subdir: str = '.') -> None: + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('base', help='Base commit') + parser.add_argument('head', help='Head commit') + args = parser.parse_args() + + for path in get_changed_files(args.base, args.head, subdir): + if path.startswith('.gitlab/linters'): + continue + for linter in linters: + linter.lint(path) + + warnings = [warning + for linter in linters + for warning in linter.warnings] + warnings = sorted(warnings, key=lambda x: (x.path, x.line_no)) + for w in warnings: + lint_failure(w.path, w.line_no, w.line_content, w.message) + + if len(warnings) > 0: + sys.exit(1) diff --git a/.gitlab/win32-init.sh b/.gitlab/win32-init.sh new file mode 100644 index 0000000000..dce9be8d16 --- /dev/null +++ b/.gitlab/win32-init.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +toolchain=`pwd`/toolchain +PATH="$toolchain/bin:/mingw64/bin:$PATH" + +if [ -d "`pwd`/cabal-cache" ]; then + cp -Rf cabal-cache $APPDATA/cabal +fi + +if [ ! -e $toolchain/bin/ghc ]; then + curl https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-x86_64-unknown-mingw32.tar.xz | tar -xJ + mv ghc-$GHC_VERSION toolchain +fi + +if [ ! -e $toolchain/bin/cabal ]; then + curl https://www.haskell.org/cabal/release/cabal-install-2.2.0.0/cabal-install-2.2.0.0-i386-unknown-mingw32.zip > /tmp/cabal.zip + unzip /tmp/cabal.zip + mv cabal.exe $toolchain/bin +fi + +if [ ! -e $toolchain/bin/happy ]; then + cabal update + cabal install happy + cp $APPDATA/cabal/bin/happy $toolchain/bin +fi + +if [ ! -e $toolchain/bin/alex ]; then + cabal update + cabal install alex + cp $APPDATA/cabal/bin/alex $toolchain/bin +fi + diff --git a/mk/detect-cpu-count.sh b/mk/detect-cpu-count.sh new file mode 100755 index 0000000000..abc47387d1 --- /dev/null +++ b/mk/detect-cpu-count.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +detect_cpu_count () { + if [ "$CPUS" = "" ]; then + # Windows standard environment variable + CPUS="$NUMBER_OF_PROCESSORS" + fi + + if [ "$CPUS" = "" ]; then + # Linux + CPUS=`getconf _NPROCESSORS_ONLN 2>/dev/null` + fi + + if [ "$CPUS" = "" ]; then + # FreeBSD + CPUS=`getconf NPROCESSORS_ONLN 2>/dev/null` + fi + + if [ "$CPUS" = "" ]; then + # nothing helped + CPUS="1" + fi +} + +detect_cpu_count +echo "$CPUS" |