From fc8335a17f14c2b8615b94303b732ae9b10d7a04 Mon Sep 17 00:00:00 2001 From: Alex Turbov Date: Tue, 7 Feb 2023 03:54:07 +0400 Subject: ci: Reduce Fedora and Debian image sizes even more Apply the approach from commit 354fdf2116 (ci: Reduce intermediate docker layers and final image size, 2023-02-01) to more stages. Also: * Use cache (prefetch metadata and packages) to reduce network I/O and speedup image build. * Use `tmpfs` to drop logs produced by the package manager. --- .gitlab/ci/docker/debian10-aarch64/Dockerfile | 29 +++- .../ci/docker/debian10-aarch64/deps_packages.lst | 90 +++++++++++++ .gitlab/ci/docker/debian10-aarch64/docker-clean | 0 .gitlab/ci/docker/debian10-aarch64/dpkg-exclude | 21 +++ .gitlab/ci/docker/debian10-aarch64/install_deps.sh | 102 +------------- .gitlab/ci/docker/debian10/Dockerfile | 62 +++++++-- .gitlab/ci/docker/debian10/deps_packages.lst | 96 ++++++++++++++ .gitlab/ci/docker/debian10/docker-clean | 0 .gitlab/ci/docker/debian10/dpkg-exclude | 21 +++ .gitlab/ci/docker/debian10/install_deps.sh | 114 +--------------- .gitlab/ci/docker/debian10/install_iwyu.sh | 11 +- .gitlab/ci/docker/debian10/install_rvm.sh | 12 +- .gitlab/ci/docker/debian10/iwyu_packages.lst | 9 ++ .gitlab/ci/docker/debian10/rvm_packages.lst | 25 ++++ .gitlab/ci/docker/fedora37/Dockerfile | 73 +++++++--- .../fedora37/clang_tidy_headers_packages.lst | 4 + .gitlab/ci/docker/fedora37/deps_packages.lst | 110 +++++++++++++++ .../docker/fedora37/install_clang_tidy_headers.sh | 12 +- .gitlab/ci/docker/fedora37/install_deps.sh | 147 +++------------------ .gitlab/ci/docker/fedora37/install_iwyu.sh | 14 +- .gitlab/ci/docker/fedora37/install_rvm.sh | 18 ++- .gitlab/ci/docker/fedora37/iwyu_packages.lst | 7 + .gitlab/ci/docker/fedora37/rvm_packages.lst | 18 +++ .gitlab/os-linux.yml | 6 +- 24 files changed, 593 insertions(+), 408 deletions(-) create mode 100644 .gitlab/ci/docker/debian10-aarch64/deps_packages.lst create mode 100644 .gitlab/ci/docker/debian10-aarch64/docker-clean create mode 100644 .gitlab/ci/docker/debian10-aarch64/dpkg-exclude create mode 100644 .gitlab/ci/docker/debian10/deps_packages.lst create mode 100644 .gitlab/ci/docker/debian10/docker-clean create mode 100644 .gitlab/ci/docker/debian10/dpkg-exclude create mode 100644 .gitlab/ci/docker/debian10/iwyu_packages.lst create mode 100644 .gitlab/ci/docker/debian10/rvm_packages.lst create mode 100644 .gitlab/ci/docker/fedora37/clang_tidy_headers_packages.lst create mode 100644 .gitlab/ci/docker/fedora37/deps_packages.lst create mode 100644 .gitlab/ci/docker/fedora37/iwyu_packages.lst create mode 100644 .gitlab/ci/docker/fedora37/rvm_packages.lst (limited to '.gitlab') diff --git a/.gitlab/ci/docker/debian10-aarch64/Dockerfile b/.gitlab/ci/docker/debian10-aarch64/Dockerfile index 2079795e99..a0687e3e30 100644 --- a/.gitlab/ci/docker/debian10-aarch64/Dockerfile +++ b/.gitlab/ci/docker/debian10-aarch64/Dockerfile @@ -1,5 +1,26 @@ -FROM arm64v8/debian:10 -MAINTAINER Brad King +# syntax=docker/dockerfile:1 -COPY install_deps.sh /root/install_deps.sh -RUN sh /root/install_deps.sh +ARG BASE_IMAGE=arm64v8/debian:10 + +FROM ${BASE_IMAGE} AS apt-cache +# Populate APT cache w/ the fresh metadata and prefetch packages. +# Use an empty `docker-clean` file to "hide" the image-provided +# file to disallow removing packages after `apt-get` operations. +RUN --mount=type=tmpfs,target=/var/log \ + --mount=type=bind,source=docker-clean,target=/etc/apt/apt.conf.d/docker-clean \ + --mount=type=bind,source=deps_packages.lst,target=/root/deps_packages.lst \ + apt-get update \ + && apt-get --download-only -y install $(grep -h '^[^#]\+$' /root/*.lst) + +FROM ${BASE_IMAGE} +LABEL maintainer="Brad King " + +RUN --mount=type=bind,source=install_deps.sh,target=/root/install_deps.sh \ + --mount=type=bind,source=deps_packages.lst,target=/root/deps_packages.lst \ + --mount=type=bind,source=dpkg-exclude,target=/etc/dpkg/dpkg.cfg.d/exclude \ + --mount=type=bind,source=docker-clean,target=/etc/apt/apt.conf.d/docker-clean \ + --mount=type=cache,from=apt-cache,source=/var/lib/apt/lists,target=/var/lib/apt/lists \ + --mount=type=cache,from=apt-cache,source=/var/cache/apt,target=/var/cache/apt,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_deps.sh diff --git a/.gitlab/ci/docker/debian10-aarch64/deps_packages.lst b/.gitlab/ci/docker/debian10-aarch64/deps_packages.lst new file mode 100644 index 0000000000..5e30e160a2 --- /dev/null +++ b/.gitlab/ci/docker/debian10-aarch64/deps_packages.lst @@ -0,0 +1,90 @@ +# Install build requirements. +libssl-dev + +# Install development tools. +g++ +curl +git + +# Install optional external build dependencies. +libarchive-dev +libbz2-dev +libcurl4-gnutls-dev +libexpat1-dev +libjsoncpp-dev +liblzma-dev +libncurses-dev +librhash-dev +libuv1-dev +libzstd-dev +zlib1g-dev + +# Install iwyu runtime deps. +clang-6.0 +libncurses6 + +# Tools needed for the test suite. +jq + +# Packages needed to test CTest. +bzr bzr-xmloutput +cvs +subversion +mercurial + +# Packages needed to test find modules. +alsa-utils +doxygen graphviz +freeglut3-dev +gnutls-dev +libarchive-dev +libblas-dev +libboost-dev +libboost-filesystem-dev +libboost-program-options-dev +libboost-python-dev +libboost-thread-dev +libbz2-dev +libcups2-dev +libcurl4-gnutls-dev +libdevil-dev +libfontconfig1-dev +libfreetype6-dev +libgdal-dev +libgif-dev +libgl1-mesa-dev +libglew-dev +libgmock-dev +libgrpc++-dev libgrpc-dev +libgsl-dev +libgtest-dev +libgtk2.0-dev +libicu-dev +libinput-dev +libjpeg-dev +libjsoncpp-dev +liblapack-dev +liblzma-dev +libmagick++-dev +libopenal-dev +libopenmpi-dev openmpi-bin +libosp-dev +libpng-dev +libpq-dev postgresql-server-dev-11 +libprotobuf-dev libprotobuf-c-dev libprotoc-dev protobuf-compiler protobuf-compiler-grpc +libsdl-dev +libsqlite3-dev +libtiff-dev +libuv1-dev +libx11-dev +libxalan-c-dev +libxerces-c-dev +libxml2-dev libxml2-utils +libxslt-dev xsltproc +openjdk-11-jdk +python2 python2-dev python-numpy pypy pypy-dev +python3 python3-dev python3-numpy pypy3 pypy3-dev python3-venv +qtbase5-dev qtbase5-dev-tools +ruby ruby-dev +swig +unixodbc-dev diff --git a/.gitlab/ci/docker/debian10-aarch64/docker-clean b/.gitlab/ci/docker/debian10-aarch64/docker-clean new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.gitlab/ci/docker/debian10-aarch64/dpkg-exclude b/.gitlab/ci/docker/debian10-aarch64/dpkg-exclude new file mode 100644 index 0000000000..60b656549c --- /dev/null +++ b/.gitlab/ci/docker/debian10-aarch64/dpkg-exclude @@ -0,0 +1,21 @@ +# Drop all man pages +path-exclude=/usr/share/man/* + +# Drop all info pages +path-exclude=/usr/share/info/* + +# Drop all README files except from the some packages +path-exclude=/usr/**/*README* +path-include=/usr/share/devscripts/templates/README.mk-build-deps +path-include=/usr/share/equivs/template/debian/README.Debian.in + +# Drop all translations +path-exclude=/usr/share/locale/*/LC_MESSAGES/*.mo + +# Drop all documentation ... +path-exclude=/usr/share/doc/* +path-exclude=/usr/share/doc-base/* +path-exclude=/usr/share/gtk-doc/* + +# Per package excludes +path-exclude=/usr/share/gnupg/help.*.txt diff --git a/.gitlab/ci/docker/debian10-aarch64/install_deps.sh b/.gitlab/ci/docker/debian10-aarch64/install_deps.sh index 7396ad7ee1..d1c8aed53c 100755 --- a/.gitlab/ci/docker/debian10-aarch64/install_deps.sh +++ b/.gitlab/ci/docker/debian10-aarch64/install_deps.sh @@ -2,104 +2,4 @@ set -e -apt-get update - -# Install build requirements. -apt-get install -y \ - libssl-dev - -# Install development tools. -apt-get install -y \ - g++ \ - curl \ - git - -# Install optional external build dependencies. -apt-get install -y \ - libarchive-dev \ - libbz2-dev \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - libjsoncpp-dev \ - liblzma-dev \ - libncurses-dev \ - librhash-dev \ - libuv1-dev \ - libzstd-dev \ - zlib1g-dev - -# Install iwyu runtime deps. -apt-get install -y \ - clang-6.0 \ - libncurses6 - -# Tools needed for the test suite. -apt-get install -y \ - jq - -# Packages needed to test CTest. -apt-get install -y \ - bzr bzr-xmloutput \ - cvs \ - subversion \ - mercurial - -# Packages needed to test find modules. -apt-get install -y \ - alsa-utils \ - doxygen graphviz \ - freeglut3-dev \ - gnutls-dev \ - libarchive-dev \ - libblas-dev \ - libboost-dev \ - libboost-filesystem-dev \ - libboost-program-options-dev \ - libboost-python-dev \ - libboost-thread-dev \ - libbz2-dev \ - libcups2-dev \ - libcurl4-gnutls-dev \ - libdevil-dev \ - libfontconfig1-dev \ - libfreetype6-dev \ - libgdal-dev \ - libgif-dev \ - libgl1-mesa-dev \ - libglew-dev \ - libgmock-dev \ - libgrpc++-dev libgrpc-dev \ - libgsl-dev \ - libgtest-dev \ - libgtk2.0-dev \ - libicu-dev \ - libinput-dev \ - libjpeg-dev \ - libjsoncpp-dev \ - liblapack-dev \ - liblzma-dev \ - libmagick++-dev \ - libopenal-dev \ - libopenmpi-dev openmpi-bin \ - libosp-dev \ - libpng-dev \ - libpq-dev postgresql-server-dev-11 \ - libprotobuf-dev libprotobuf-c-dev libprotoc-dev protobuf-compiler protobuf-compiler-grpc \ - libsdl-dev \ - libsqlite3-dev \ - libtiff-dev \ - libuv1-dev \ - libx11-dev \ - libxalan-c-dev \ - libxerces-c-dev \ - libxml2-dev libxml2-utils \ - libxslt-dev xsltproc \ - openjdk-11-jdk \ - python2 python2-dev python-numpy pypy pypy-dev \ - python3 python3-dev python3-numpy pypy3 pypy3-dev python3-venv \ - qtbase5-dev qtbase5-dev-tools \ - ruby ruby-dev \ - swig \ - unixodbc-dev - -apt-get clean +apt-get install -y $(grep '^[^#]\+$' /root/deps_packages.lst) diff --git a/.gitlab/ci/docker/debian10/Dockerfile b/.gitlab/ci/docker/debian10/Dockerfile index 2d3ae3aff8..d86642876c 100644 --- a/.gitlab/ci/docker/debian10/Dockerfile +++ b/.gitlab/ci/docker/debian10/Dockerfile @@ -1,20 +1,58 @@ -FROM debian:10 AS iwyu-build -MAINTAINER Ben Boeckel +# syntax=docker/dockerfile:1 -COPY install_iwyu.sh /root/install_iwyu.sh -RUN sh /root/install_iwyu.sh +ARG BASE_IMAGE=debian:10 -FROM debian:10 AS rvm-build -MAINTAINER Ben Boeckel +FROM ${BASE_IMAGE} AS apt-cache +# Populate APT cache w/ the fresh metadata and prefetch packages. +# Use an empty `docker-clean` file to "hide" the image-provided +# file to disallow removing packages after `apt-get` operations. +RUN --mount=type=tmpfs,target=/var/log \ + --mount=type=bind,source=docker-clean,target=/etc/apt/apt.conf.d/docker-clean \ + --mount=type=bind,source=deps_packages.lst,target=/root/deps_packages.lst \ + --mount=type=bind,source=iwyu_packages.lst,target=/root/iwyu_packages.lst \ + --mount=type=bind,source=rvm_packages.lst,target=/root/rvm_packages.lst \ + apt-get update \ + && apt-get --download-only -y install $(grep -h '^[^#]\+$' /root/*.lst) -COPY install_rvm.sh /root/install_rvm.sh -RUN sh /root/install_rvm.sh -FROM debian:10 -MAINTAINER Ben Boeckel +FROM ${BASE_IMAGE} AS iwyu-build +LABEL maintainer="Ben Boeckel " -COPY install_deps.sh /root/install_deps.sh -RUN sh /root/install_deps.sh +RUN --mount=type=bind,source=install_iwyu.sh,target=/root/install_iwyu.sh \ + --mount=type=bind,source=iwyu_packages.lst,target=/root/iwyu_packages.lst \ + --mount=type=bind,source=docker-clean,target=/etc/apt/apt.conf.d/docker-clean \ + --mount=type=cache,from=apt-cache,source=/var/lib/apt/lists,target=/var/lib/apt/lists \ + --mount=type=cache,from=apt-cache,source=/var/cache/apt,target=/var/cache/apt,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_iwyu.sh + + +FROM ${BASE_IMAGE} AS rvm-build +LABEL maintainer="Ben Boeckel " + +RUN --mount=type=bind,source=install_rvm.sh,target=/root/install_rvm.sh \ + --mount=type=bind,source=rvm_packages.lst,target=/root/rvm_packages.lst \ + --mount=type=bind,source=docker-clean,target=/etc/apt/apt.conf.d/docker-clean \ + --mount=type=cache,from=apt-cache,source=/var/lib/apt/lists,target=/var/lib/apt/lists \ + --mount=type=cache,from=apt-cache,source=/var/cache/apt,target=/var/cache/apt,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_rvm.sh + + +FROM ${BASE_IMAGE} +LABEL maintainer="Ben Boeckel " + +RUN --mount=type=bind,source=install_deps.sh,target=/root/install_deps.sh \ + --mount=type=bind,source=deps_packages.lst,target=/root/deps_packages.lst \ + --mount=type=bind,source=dpkg-exclude,target=/etc/dpkg/dpkg.cfg.d/exclude \ + --mount=type=bind,source=docker-clean,target=/etc/apt/apt.conf.d/docker-clean \ + --mount=type=cache,from=apt-cache,source=/var/lib/apt/lists,target=/var/lib/apt/lists \ + --mount=type=cache,from=apt-cache,source=/var/cache/apt,target=/var/cache/apt,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_deps.sh RUN --mount=type=bind,from=iwyu-build,source=/root,target=/root \ tar -C / -xf /root/iwyu.tar \ diff --git a/.gitlab/ci/docker/debian10/deps_packages.lst b/.gitlab/ci/docker/debian10/deps_packages.lst new file mode 100644 index 0000000000..3df41f5bc5 --- /dev/null +++ b/.gitlab/ci/docker/debian10/deps_packages.lst @@ -0,0 +1,96 @@ +# Install build requirements. +libssl-dev + +# Install development tools. +g++ +curl +git + +# Install optional external build dependencies. +libarchive-dev +libbz2-dev +libcurl4-gnutls-dev +libexpat1-dev +libjsoncpp-dev +liblzma-dev +libncurses-dev +librhash-dev +libuv1-dev +libzstd-dev +zlib1g-dev + +# Install iwyu runtime deps. +clang-6.0 +libncurses6 + +# Tools needed for the test suite. +jq + +# Packages needed to test CTest. +bzr bzr-xmloutput +cvs +subversion +mercurial + +# Install swift runtime deps. +libncurses5 + +# Packages needed to test find modules. +alsa-utils +doxygen graphviz +freeglut3-dev +gnutls-dev +libarchive-dev +libblas-dev +libboost-dev +libboost-filesystem-dev +libboost-program-options-dev +libboost-python-dev +libboost-thread-dev +libbz2-dev +libcups2-dev +libcurl4-gnutls-dev +libdevil-dev +libfontconfig1-dev +libfreetype6-dev +libgdal-dev +libgif-dev +libgl1-mesa-dev +libglew-dev +libgmock-dev +libgrpc++-dev libgrpc-dev +libgsl-dev +libgtest-dev +libgtk2.0-dev +libicu-dev +libinput-dev +libjpeg-dev +libjsoncpp-dev +liblapack-dev +liblzma-dev +libmagick++-dev +libopenal-dev +libopenmpi-dev openmpi-bin +libosp-dev +libpng-dev +libpq-dev postgresql-server-dev-11 +libprotobuf-dev libprotobuf-c-dev libprotoc-dev protobuf-compiler protobuf-compiler-grpc +libsdl-dev +libsqlite3-dev +libtiff-dev +libuv1-dev +libx11-dev +libxalan-c-dev +libxerces-c-dev +libxml2-dev libxml2-utils +libxslt-dev xsltproc +openjdk-11-jdk +python2 python2-dev python-numpy pypy pypy-dev +python3 python3-dev python3-numpy pypy3 pypy3-dev python3-venv +qtbase5-dev qtbase5-dev-tools +ruby ruby-dev +swig +unixodbc-dev + +# CMake_TEST_FindPython_IronPython +libmono-system-windows-forms4.0-cil diff --git a/.gitlab/ci/docker/debian10/docker-clean b/.gitlab/ci/docker/debian10/docker-clean new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.gitlab/ci/docker/debian10/dpkg-exclude b/.gitlab/ci/docker/debian10/dpkg-exclude new file mode 100644 index 0000000000..60b656549c --- /dev/null +++ b/.gitlab/ci/docker/debian10/dpkg-exclude @@ -0,0 +1,21 @@ +# Drop all man pages +path-exclude=/usr/share/man/* + +# Drop all info pages +path-exclude=/usr/share/info/* + +# Drop all README files except from the some packages +path-exclude=/usr/**/*README* +path-include=/usr/share/devscripts/templates/README.mk-build-deps +path-include=/usr/share/equivs/template/debian/README.Debian.in + +# Drop all translations +path-exclude=/usr/share/locale/*/LC_MESSAGES/*.mo + +# Drop all documentation ... +path-exclude=/usr/share/doc/* +path-exclude=/usr/share/doc-base/* +path-exclude=/usr/share/gtk-doc/* + +# Per package excludes +path-exclude=/usr/share/gnupg/help.*.txt diff --git a/.gitlab/ci/docker/debian10/install_deps.sh b/.gitlab/ci/docker/debian10/install_deps.sh index 3e5622ba5d..a00e322cd8 100755 --- a/.gitlab/ci/docker/debian10/install_deps.sh +++ b/.gitlab/ci/docker/debian10/install_deps.sh @@ -2,113 +2,8 @@ set -e -apt-get update +apt-get install -y $(grep '^[^#]\+$' /root/deps_packages.lst) -# Install build requirements. -apt-get install -y \ - libssl-dev - -# Install development tools. -apt-get install -y \ - g++ \ - curl \ - git - -# Install optional external build dependencies. -apt-get install -y \ - libarchive-dev \ - libbz2-dev \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - libjsoncpp-dev \ - liblzma-dev \ - libncurses-dev \ - librhash-dev \ - libuv1-dev \ - libzstd-dev \ - zlib1g-dev - -# Install iwyu runtime deps. -apt-get install -y \ - clang-6.0 \ - libncurses6 - -# Tools needed for the test suite. -apt-get install -y \ - jq - -# Packages needed to test CTest. -apt-get install -y \ - bzr bzr-xmloutput \ - cvs \ - subversion \ - mercurial - -# Install swift runtime deps. -apt-get install -y \ - libncurses5 - -# Packages needed to test find modules. -apt-get install -y \ - alsa-utils \ - doxygen graphviz \ - freeglut3-dev \ - gnutls-dev \ - libarchive-dev \ - libblas-dev \ - libboost-dev \ - libboost-filesystem-dev \ - libboost-program-options-dev \ - libboost-python-dev \ - libboost-thread-dev \ - libbz2-dev \ - libcups2-dev \ - libcurl4-gnutls-dev \ - libdevil-dev \ - libfontconfig1-dev \ - libfreetype6-dev \ - libgdal-dev \ - libgif-dev \ - libgl1-mesa-dev \ - libglew-dev \ - libgmock-dev \ - libgrpc++-dev libgrpc-dev \ - libgsl-dev \ - libgtest-dev \ - libgtk2.0-dev \ - libicu-dev \ - libinput-dev \ - libjpeg-dev \ - libjsoncpp-dev \ - liblapack-dev \ - liblzma-dev \ - libmagick++-dev \ - libopenal-dev \ - libopenmpi-dev openmpi-bin \ - libosp-dev \ - libpng-dev \ - libpq-dev postgresql-server-dev-11 \ - libprotobuf-dev libprotobuf-c-dev libprotoc-dev protobuf-compiler protobuf-compiler-grpc \ - libsdl-dev \ - libsqlite3-dev \ - libtiff-dev \ - libuv1-dev \ - libx11-dev \ - libxalan-c-dev \ - libxerces-c-dev \ - libxml2-dev libxml2-utils \ - libxslt-dev xsltproc \ - openjdk-11-jdk \ - python2 python2-dev python-numpy pypy pypy-dev \ - python3 python3-dev python3-numpy pypy3 pypy3-dev python3-venv \ - qtbase5-dev qtbase5-dev-tools \ - ruby ruby-dev \ - swig \ - unixodbc-dev - -# CMake_TEST_FindPython_IronPython -apt-get install -y \ - libmono-system-windows-forms4.0-cil curl -L -O https://github.com/IronLanguages/ironpython2/releases/download/ipy-2.7.10/ironpython_2.7.10.deb echo 'e1aceec1d49ffa66e9059a52168a734999dcccc50164a60e2936649cae698f3e ironpython_2.7.10.deb' > ironpython.sha256sum sha256sum --check ironpython.sha256sum @@ -116,8 +11,5 @@ dpkg -i ironpython_2.7.10.deb rm ironpython_2.7.10.deb ironpython.sha256sum # Perforce -curl -L -O https://www.perforce.com/downloads/perforce/r21.2/bin.linux26x86_64/helix-core-server.tgz -tar -C /usr/local/bin -xvzf helix-core-server.tgz -- p4 p4d -rm helix-core-server.tgz - -apt-get clean +curl -L https://www.perforce.com/downloads/perforce/r21.2/bin.linux26x86_64/helix-core-server.tgz -o - \ + | tar -C /usr/local/bin -xvzf - -- p4 p4d diff --git a/.gitlab/ci/docker/debian10/install_iwyu.sh b/.gitlab/ci/docker/debian10/install_iwyu.sh index 260570e8ec..4814a7161e 100755 --- a/.gitlab/ci/docker/debian10/install_iwyu.sh +++ b/.gitlab/ci/docker/debian10/install_iwyu.sh @@ -3,16 +3,7 @@ set -e # Install development tools. -apt-get update -apt-get install -y \ - clang-6.0 \ - libclang-6.0-dev \ - llvm-6.0-dev \ - libz-dev \ - g++ \ - cmake \ - ninja-build \ - git +apt-get install -y $(grep '^[^#]\+$' /root/iwyu_packages.lst) cd /root git clone "https://github.com/include-what-you-use/include-what-you-use.git" diff --git a/.gitlab/ci/docker/debian10/install_rvm.sh b/.gitlab/ci/docker/debian10/install_rvm.sh index 0ebc74653d..c6fff70d02 100755 --- a/.gitlab/ci/docker/debian10/install_rvm.sh +++ b/.gitlab/ci/docker/debian10/install_rvm.sh @@ -2,11 +2,7 @@ set -e -apt-get update -apt-get install -y \ - curl \ - gnupg2 \ - procps +apt-get install -y $(grep '^[^#]\+$' /root/rvm_packages.lst) gpg2 --keyserver hkps://keyserver.ubuntu.com \ --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \ @@ -17,4 +13,8 @@ curl -sSL https://get.rvm.io | bash -s stable # keep version in sync with `env_debian*_ninja.sh` /usr/local/rvm/bin/rvm install ruby-2.7.0 -tar -C /usr/local -cf /root/rvm.tar rvm +for p in archives examples gem-cache log src; do + touch /usr/local/rvm/${p}/.tar_exclude +done + +tar -C /usr/local --exclude-tag-under=.tar_exclude -cf /root/rvm.tar rvm diff --git a/.gitlab/ci/docker/debian10/iwyu_packages.lst b/.gitlab/ci/docker/debian10/iwyu_packages.lst new file mode 100644 index 0000000000..9e291c9b04 --- /dev/null +++ b/.gitlab/ci/docker/debian10/iwyu_packages.lst @@ -0,0 +1,9 @@ +# Install development tools. +clang-6.0 +libclang-6.0-dev +llvm-6.0-dev +libz-dev +g++ +cmake +ninja-build +git diff --git a/.gitlab/ci/docker/debian10/rvm_packages.lst b/.gitlab/ci/docker/debian10/rvm_packages.lst new file mode 100644 index 0000000000..80f079cbf2 --- /dev/null +++ b/.gitlab/ci/docker/debian10/rvm_packages.lst @@ -0,0 +1,25 @@ +autoconf +automake +bison +bzip2 +curl +g++ +gawk +gcc +gnupg2 +libc6-dev +libffi-dev +libgdbm-dev +libgmp-dev +libncurses5-dev +libreadline-dev +libsqlite3-dev +libssl-dev +libtool +libyaml-dev +make +patch +pkg-config +procps +sqlite3 +zlib1g-dev diff --git a/.gitlab/ci/docker/fedora37/Dockerfile b/.gitlab/ci/docker/fedora37/Dockerfile index 1ba9e373eb..b36a17ed24 100644 --- a/.gitlab/ci/docker/fedora37/Dockerfile +++ b/.gitlab/ci/docker/fedora37/Dockerfile @@ -1,26 +1,67 @@ -FROM fedora:37 AS rvm-build -MAINTAINER Ben Boeckel +# syntax=docker/dockerfile:1 -COPY install_rvm.sh /root/install_rvm.sh -RUN sh /root/install_rvm.sh +ARG BASE_IMAGE=fedora:37 -FROM fedora:37 AS clang-tidy-headers -MAINTAINER Kyle Edwards +FROM ${BASE_IMAGE} AS dnf-cache +# Populate DNF cache w/ the fresh metadata and prefetch packages. +RUN --mount=type=bind,source=clang_tidy_headers_packages.lst,target=/root/clang_tidy_headers_packages.lst \ + --mount=type=bind,source=deps_packages.lst,target=/root/deps_packages.lst \ + --mount=type=bind,source=iwyu_packages.lst,target=/root/iwyu_packages.lst \ + --mount=type=bind,source=rvm_packages.lst,target=/root/rvm_packages.lst \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + dnf install \ + --setopt=install_weak_deps=False \ + --setopt=fastestmirror=True \ + --setopt=max_parallel_downloads=10 \ + --downloadonly \ + -y \ + $(grep -h '^[^#]\+$' /root/*.lst) -COPY install_clang_tidy_headers.sh /root/install_clang_tidy_headers.sh -RUN sh /root/install_clang_tidy_headers.sh -FROM fedora:37 AS iwyu-build -MAINTAINER Kyle Edwards +FROM ${BASE_IMAGE} AS rvm-build +LABEL maintainer="Ben Boeckel " -COPY install_iwyu.sh /root/install_iwyu.sh -RUN sh /root/install_iwyu.sh +RUN --mount=type=bind,source=install_rvm.sh,target=/root/install_rvm.sh \ + --mount=type=bind,source=rvm_packages.lst,target=/root/rvm_packages.lst \ + --mount=type=cache,from=dnf-cache,source=/var/cache/dnf,target=/var/cache/dnf,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_rvm.sh -FROM fedora:37 -MAINTAINER Ben Boeckel -COPY install_deps.sh /root/install_deps.sh -RUN sh /root/install_deps.sh +FROM ${BASE_IMAGE} AS clang-tidy-headers +LABEL maintainer="Kyle Edwards " + +RUN --mount=type=bind,source=install_clang_tidy_headers.sh,target=/root/install_clang_tidy_headers.sh \ + --mount=type=bind,source=clang_tidy_headers_packages.lst,target=/root/clang_tidy_headers_packages.lst \ + --mount=type=cache,from=dnf-cache,source=/var/cache/dnf,target=/var/cache/dnf,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_clang_tidy_headers.sh + + +FROM ${BASE_IMAGE} AS iwyu-build +LABEL maintainer="Kyle Edwards " + +RUN --mount=type=bind,source=install_iwyu.sh,target=/root/install_iwyu.sh \ + --mount=type=bind,source=iwyu_packages.lst,target=/root/iwyu_packages.lst \ + --mount=type=cache,from=dnf-cache,source=/var/cache/dnf,target=/var/cache/dnf,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_iwyu.sh + + +FROM ${BASE_IMAGE} +LABEL maintainer="Ben Boeckel " + +RUN --mount=type=bind,source=install_deps.sh,target=/root/install_deps.sh \ + --mount=type=bind,source=deps_packages.lst,target=/root/deps_packages.lst \ + --mount=type=cache,from=dnf-cache,source=/var/cache/dnf,target=/var/cache/dnf,sharing=private \ + --mount=type=cache,target=/var/cache/pip \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/tmp \ + sh /root/install_deps.sh RUN --mount=type=bind,from=rvm-build,source=/root,target=/root \ tar -C /usr/local -xf /root/rvm.tar diff --git a/.gitlab/ci/docker/fedora37/clang_tidy_headers_packages.lst b/.gitlab/ci/docker/fedora37/clang_tidy_headers_packages.lst new file mode 100644 index 0000000000..fe86105942 --- /dev/null +++ b/.gitlab/ci/docker/fedora37/clang_tidy_headers_packages.lst @@ -0,0 +1,4 @@ +dnf-command(download) +rpm-build +python3-devel +clang-tools-extra diff --git a/.gitlab/ci/docker/fedora37/deps_packages.lst b/.gitlab/ci/docker/fedora37/deps_packages.lst new file mode 100644 index 0000000000..9ce80075ec --- /dev/null +++ b/.gitlab/ci/docker/fedora37/deps_packages.lst @@ -0,0 +1,110 @@ +# Install build requirements. +ncurses-devel +openssl-devel +qt5-qtbase-devel +qt6-qtbase-devel + +# Install development tools. +clang +clang-tools-extra +compiler-rt +flang +flang-devel +gcc-c++ +git-core +make + +# Install optional external build dependencies. +bzip2-devel +expat-devel +jsoncpp-devel +libarchive-devel +libcurl-devel +libuv-devel +libuv-devel +libzstd-devel +rhash-devel +xz-devel +zlib-devel + +# Install documentation tools. +python3-sphinx +texinfo +qt5-qttools-devel +qt6-qttools-devel + +# Install lint tools. +clang-analyzer +codespell + +# Tools needed for the test suite. +findutils +file +jq +which + +# Packages needed to test CTest. +breezy +subversion +mercurial + +# Packages needed to test CPack. +rpm-build + +# Packages needed to test find modules. +alsa-lib-devel +blas-devel +boost-devel boost-python3-devel +bzip2-devel +cups-devel +DevIL-devel +doxygen +expat-devel +fontconfig-devel +freeglut-devel +freetype-devel +gdal-devel +gettext +giflib-devel +glew-devel +gmock +gnutls-devel +grpc-devel grpc-plugins +gsl-devel +gtest-devel +gtk2-devel +ImageMagick-c++-devel +java-11-openjdk-devel +jsoncpp-devel +lapack-devel +libarchive-devel +libcurl-devel +libicu-devel +libinput-devel systemd-devel +libjpeg-turbo-devel +libpng-devel +opensp-devel +postgresql-server-devel +libtiff-devel +libuv-devel +libxml2-devel +libxslt-devel +mpich-devel +openal-soft-devel +openmpi-devel +patch +perl +protobuf-devel protobuf-c-devel protobuf-lite-devel +pypy2 pypy2-devel +pypy3 pypy3-devel +python2 python2-devel +python3 python3-devel python3-numpy +python3-jsmin python3-jsonschema +ruby rubygems ruby-devel +SDL-devel +sqlite-devel +swig +unixODBC-devel +xalan-c-devel +xerces-c-devel +xz-devel diff --git a/.gitlab/ci/docker/fedora37/install_clang_tidy_headers.sh b/.gitlab/ci/docker/fedora37/install_clang_tidy_headers.sh index b9883f4039..200fa1e501 100755 --- a/.gitlab/ci/docker/fedora37/install_clang_tidy_headers.sh +++ b/.gitlab/ci/docker/fedora37/install_clang_tidy_headers.sh @@ -4,11 +4,13 @@ set -e # Packages for building the clang-tidy plugin. # TODO: Upstream this as a proper Fedora package. -dnf install --setopt=install_weak_deps=False -y \ - 'dnf-command(download)' \ - rpm-build \ - python3-devel \ - clang-tools-extra +dnf install \ + --setopt=install_weak_deps=False \ + --setopt=fastestmirror=True \ + --setopt=max_parallel_downloads=10 \ + -y \ + $(grep '^[^#]\+$' /root/clang_tidy_headers_packages.lst) + clang_source_rpm=$(rpm -q --queryformat '%{SOURCERPM}' clang-tools-extra) clang_version=$(rpm -q --queryformat '%{VERSION}' clang-tools-extra) dnf download --source -y clang diff --git a/.gitlab/ci/docker/fedora37/install_deps.sh b/.gitlab/ci/docker/fedora37/install_deps.sh index 35089299a1..cd2701ebee 100755 --- a/.gitlab/ci/docker/fedora37/install_deps.sh +++ b/.gitlab/ci/docker/fedora37/install_deps.sh @@ -2,135 +2,30 @@ set -e -# Install build requirements. -dnf install --setopt=install_weak_deps=False -y \ - ncurses-devel \ - openssl-devel \ - qt5-qtbase-devel \ - qt6-qtbase-devel +dnf install \ + --setopt=install_weak_deps=False \ + --setopt=fastestmirror=True \ + --setopt=max_parallel_downloads=10 \ + -y \ + $(grep '^[^#]\+$' /root/deps_packages.lst) -# Install development tools. -dnf install --setopt=install_weak_deps=False -y \ - clang \ - clang-tools-extra \ - compiler-rt \ - flang \ - flang-devel \ - gcc-c++ \ - git-core \ - make - -# Install optional external build dependencies. -dnf install --setopt=install_weak_deps=False -y \ - bzip2-devel \ - expat-devel \ - jsoncpp-devel \ - libarchive-devel \ - libcurl-devel \ - libuv-devel \ - libuv-devel \ - libzstd-devel \ - rhash-devel \ - xz-devel \ - zlib-devel - -# Install documentation tools. -dnf install --setopt=install_weak_deps=False -y \ - python3-sphinx \ - texinfo \ - qt5-qttools-devel \ - qt6-qttools-devel - -# Install lint tools. -dnf install --setopt=install_weak_deps=False -y \ - clang-analyzer \ - codespell - -# Tools needed for the test suite. -dnf install --setopt=install_weak_deps=False -y \ - findutils \ - file \ - jq \ - which - -# Packages needed to test CTest. -dnf install --setopt=install_weak_deps=False -y \ - breezy \ - subversion \ - mercurial - -# Packages needed to test CPack. -dnf install --setopt=install_weak_deps=False -y \ - rpm-build +# Fedora no longer packages python2 numpy. +curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o - | python2 +pip2.7 install --disable-pip-version-check --no-input --no-compile --cache-dir /var/cache/pip numpy -# Packages needed to test find modules. -dnf install --setopt=install_weak_deps=False -y \ - alsa-lib-devel \ - blas-devel \ - boost-devel boost-python3-devel \ - bzip2-devel \ - cups-devel \ - DevIL-devel \ - doxygen \ - expat-devel \ - fontconfig-devel \ - freeglut-devel \ - freetype-devel \ - gdal-devel \ - gettext \ - giflib-devel \ - glew-devel \ - gmock \ - gnutls-devel \ - grpc-devel grpc-plugins \ - gsl-devel \ - gtest-devel \ - gtk2-devel \ - ImageMagick-c++-devel \ - java-11-openjdk-devel \ - jsoncpp-devel \ - lapack-devel \ - libarchive-devel \ - libcurl-devel \ - libicu-devel \ - libinput-devel systemd-devel \ - libjpeg-turbo-devel \ - libpng-devel \ - opensp-devel \ - postgresql-server-devel \ - libtiff-devel \ - libuv-devel \ - libxml2-devel \ - libxslt-devel \ - mpich-devel \ - openal-soft-devel \ - openmpi-devel \ - patch \ - perl \ - protobuf-devel protobuf-c-devel protobuf-lite-devel \ - pypy2 pypy2-devel \ - pypy3 pypy3-devel \ - python2 python2-devel \ - python3 python3-devel python3-numpy \ - python3-jsmin python3-jsonschema \ - ruby rubygems ruby-devel \ - SDL-devel \ - sqlite-devel \ - swig \ - unixODBC-devel \ - xalan-c-devel \ - xerces-c-devel \ - xz-devel +# Remove demos and Python2 tests +for p in Demo test; do + rm -rf /usr/lib64/python2.7/${p} +done -dnf clean all +# Remove tests for numpy +for v in 2.7 3.11; do + find /usr/lib64/python${v}/site-packages/numpy -type d -a -name tests -exec rm -rf {} + +done -# Fedora no longer packages python2 numpy. -curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py -python2 get-pip.py -rm get-pip.py -pip2.7 install numpy +# Remove some other packages tests +find /usr/lib64/python3.11/site-packages/breezy -type d -a -name tests -exec rm -rf {} + # Perforce -curl -L -O https://www.perforce.com/downloads/perforce/r21.2/bin.linux26x86_64/helix-core-server.tgz -tar -C /usr/local/bin -xvzf helix-core-server.tgz -- p4 p4d -rm helix-core-server.tgz +curl -L https://www.perforce.com/downloads/perforce/r21.2/bin.linux26x86_64/helix-core-server.tgz -o - \ + | tar -C /usr/local/bin -xvzf - -- p4 p4d diff --git a/.gitlab/ci/docker/fedora37/install_iwyu.sh b/.gitlab/ci/docker/fedora37/install_iwyu.sh index 714bcc05c7..684e355058 100755 --- a/.gitlab/ci/docker/fedora37/install_iwyu.sh +++ b/.gitlab/ci/docker/fedora37/install_iwyu.sh @@ -3,14 +3,12 @@ set -e # Install development tools. -dnf install --setopt=install_weak_deps=False -y \ - clang-devel \ - llvm-devel \ - zlib-devel \ - g++ \ - cmake \ - ninja-build \ - git +dnf install \ + --setopt=install_weak_deps=False \ + --setopt=fastestmirror=True \ + --setopt=max_parallel_downloads=10 \ + -y \ + $(grep '^[^#]\+$' /root/iwyu_packages.lst) cd /root git clone "https://github.com/include-what-you-use/include-what-you-use.git" diff --git a/.gitlab/ci/docker/fedora37/install_rvm.sh b/.gitlab/ci/docker/fedora37/install_rvm.sh index 0011f877f1..10e7545cf5 100755 --- a/.gitlab/ci/docker/fedora37/install_rvm.sh +++ b/.gitlab/ci/docker/fedora37/install_rvm.sh @@ -2,18 +2,24 @@ set -e +dnf install \ + --setopt=install_weak_deps=False \ + --setopt=fastestmirror=True \ + --setopt=max_parallel_downloads=10 \ + -y \ + $(grep '^[^#]\+$' /root/rvm_packages.lst) + gpg2 --keyserver hkps://keyserver.ubuntu.com \ --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \ 7D2BAF1CF37B13E2069D6956105BD0E739499BDB -dnf install --setopt=install_weak_deps=False -y \ - findutils \ - procps \ - which - curl -sSL https://get.rvm.io | bash -s stable # keep version in sync with `env_fedora*_makefiles.cmake` /usr/local/rvm/bin/rvm install ruby-3.0.4 -tar -C /usr/local -cf /root/rvm.tar rvm +for p in archives examples gem-cache log src; do + touch /usr/local/rvm/${p}/.tar_exclude +done + +tar -C /usr/local --exclude-tag-under=.tar_exclude -cf /root/rvm.tar rvm diff --git a/.gitlab/ci/docker/fedora37/iwyu_packages.lst b/.gitlab/ci/docker/fedora37/iwyu_packages.lst new file mode 100644 index 0000000000..e3551bd315 --- /dev/null +++ b/.gitlab/ci/docker/fedora37/iwyu_packages.lst @@ -0,0 +1,7 @@ +clang-devel +llvm-devel +zlib-devel +g++ +cmake +ninja-build +git diff --git a/.gitlab/ci/docker/fedora37/rvm_packages.lst b/.gitlab/ci/docker/fedora37/rvm_packages.lst new file mode 100644 index 0000000000..1dc852e147 --- /dev/null +++ b/.gitlab/ci/docker/fedora37/rvm_packages.lst @@ -0,0 +1,18 @@ +autoconf +automake +bison +bzip2 +findutils +gcc-c++ +glibc-devel +libffi-devel +libtool +libyaml-devel +make +openssl-devel +patch +procps +readline-devel +sqlite-devel +which +zlib-devel diff --git a/.gitlab/os-linux.yml b/.gitlab/os-linux.yml index 9d7be0e0a9..8ab3fad3cd 100644 --- a/.gitlab/os-linux.yml +++ b/.gitlab/os-linux.yml @@ -45,7 +45,7 @@ ### Debian .debian10: - image: "kitware/cmake:ci-debian10-x86_64-2023-02-06" + image: "kitware/cmake:ci-debian10-x86_64-2023-02-07" variables: GIT_CLONE_PATH: "$CI_BUILDS_DIR/cmake ci" @@ -60,7 +60,7 @@ CMAKE_CI_NO_INSTALL: 1 .debian10_aarch64: - image: "kitware/cmake:ci-debian10-aarch64-2023-01-26" + image: "kitware/cmake:ci-debian10-aarch64-2023-02-07" variables: GIT_CLONE_PATH: "$CI_BUILDS_DIR/cmake ci" @@ -69,7 +69,7 @@ ### Fedora .fedora37: - image: "kitware/cmake:ci-fedora37-x86_64-2023-02-06" + image: "kitware/cmake:ci-fedora37-x86_64-2023-02-07" variables: GIT_CLONE_PATH: "$CI_BUILDS_DIR/cmake ci/long file name for testing purposes" -- cgit v1.2.1