From 260649de0472e90be665ae2c442c4435e125b022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Wed, 4 Aug 2021 21:13:04 -0700 Subject: Misc improvements to the Gitlab-CI pipeline for MariaDB - Add new Ninja and Clang build jobs. This helps to ensure those toolchains also work in addition to default CMake/gcc. - Generate dependencies.dot/png to illustrate the CMake/Make/Ninja build dependencies. Viewing this image and identifying bottle necks in parallelism can help make the build run faster. - Enable CUnit tests now as they are fixed on 10.6 (MDEV-25820). - Limit parallel builds to 2 CPUs (full parallelism needs MDEV-25968) on CMake/Make. Now only the Ninja builds run full parallel builds as only Ninja is smart enough to prevent builds failing on resource over-consumption. - Enable Gitlab-CI cache for job 'centos8' for ccache so that it builds faster. Don't use Gitlab-CI cache for other jobs, as it would too easily use up all free tier storage on Gitlab.com and force users to get a paid account just for MariaDB builds. - On other jobs clean away ccache, as it only had a 5% hit rate on single builds with no downloaded cache. - Dump full database contents during the test install so that one can use diff to compare the database contents at different stages and thus track/debug potential bugs in mariadb-install-db and mariadb-upgrade code. Bugfixes: - Zero out ccache stats before each run so that 'ccache -s' would actually show the stats for the latest run. --- .gitlab-ci.yml | 201 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 152 insertions(+), 49 deletions(-) (limited to '.gitlab-ci.yml') diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a1bf5bff71..cd22fc5dd8c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,8 +37,22 @@ image: fedora:33 # many components that are otherwise slow to build. variables: CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF" - # Major version dictates which branches share the same ccache + # Major version dictates which branches share the same ccache. E.g. 10.6-abc + # and 10.6-xyz will have the same cache. MARIADB_MAJOR_VERSION: "10.6" + # NOTE! Currently ccache is only used on the Centos8 build. As each job has + # sufficiently different environments they are unable to benefit from each + # other's ccaches. As each build generates about 1 GB of ccache, having + # multiple caches would quickly consume all free storage on Gitlab-CI and + # grind all builds to a halt. Also the network overhead of download/upload + # decreases the benefit of ccache in Gitlab-CI, and current cache:when and + # cache:policy are not flexible enough to have a system where the cache is + # uploaded only once a week and not on every build. Having ccache on at least + # one build still helps ensure that ccache compatibility is at least tested + # and if the Centos 8 build is always significantly faster than all other + # builds (e.g. on self-hosted Gitlab instances) then users would at least be + # able to discover it. + # # Most steps don't need the source code, only artifacts GIT_STRATEGY: none # Hack to satisfy directory name length requirement by CPackRPM in CMake 3.x @@ -65,23 +79,21 @@ fedora: GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: normal script: - - yum install -y yum-utils rpm-build ccache openssl-devel - - source /etc/profile.d/ccache.sh - - export CCACHE_DIR="$(pwd)/.ccache"; ccache -s + - yum install -y yum-utils rpm-build openssl-devel graphviz # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm # This repository does not have any .spec files, so install dependencies based on Fedora spec file - yum-builddep -y mariadb-server - mkdir builddir; cd builddir - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log - - eatmydata make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log - # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running - # multi-proc, needs more debugging - - make test || true # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820 - # - make test-force || true # mysql-test-runner takes too long, run it in a separate job instead + - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot + - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just + # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968 + - make test + # - make test-force # mysql-test-runner takes too long, run MTR in a separate job instead - *rpm_listfiles - mkdir ../rpm; mv *.rpm ../rpm - - ccache -s artifacts: when: always # Must be able to see logs paths: @@ -89,14 +101,74 @@ fedora: - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log - rpm - builddir/_CPack_Packages/Linux/RPM/SPECS/ - cache: - key: $MARIADB_MAJOR_VERSION-$CI_JOB_NAME + - dependencies.dot + - dependencies.png + +fedora-ninja: + stage: build + variables: + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: normal + script: + - yum install -y yum-utils rpm-build openssl-devel graphviz ninja-build + # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway + - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm + # This repository does not have any .spec files, so install dependencies based on Fedora spec file + - yum-builddep -y mariadb-server + - mkdir builddir; cd builddir + - cmake -DRPM=generic $CMAKE_FLAGS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -G Ninja .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + - ninja -t graph > ../dependencies.dot && dot -Tpng -o ../dependencies.png ../dependencies.dot + - eatmydata ninja package --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + # Ninja builds are not affected by bug https://jira.mariadb.org/browse/MDEV-25968 + - ninja test + - *rpm_listfiles + - mkdir ../rpm; mv *.rpm ../rpm + artifacts: + when: always # Must be able to see logs paths: - - .ccache - # policy: pull - # @TODO: It would be enough to only download the cache. There is no need for - # every job to upload a new cache every time. A monthly or weekly scheduled - # run could update the cache for all other builds to us. + - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + - rpm + - builddir/_CPack_Packages/Linux/RPM/SPECS/ + - dependencies.dot + - dependencies.png + +fedora-clang: + stage: build + variables: + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: normal + script: + - yum install -y yum-utils rpm-build openssl-devel graphviz clang + # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway + - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm + # This repository does not have any .spec files, so install dependencies based on Fedora spec file + - yum-builddep -y mariadb-server + - mkdir builddir; cd builddir + - export CXX=${CXX:-clang++} + - export CC=${CC:-clang} + - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++} + - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang} + - export CFLAGS='-Wno-unused-command-line-argument' + - export CXXFLAGS='-Wno-unused-command-line-argument' + - cmake -DRPM=generic $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot + - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just + # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968 + - make test + # - make test-force # mysql-test-runner takes too long, run MTr in a separate job instead + - *rpm_listfiles + - mkdir ../rpm; mv *.rpm ../rpm + artifacts: + when: always # Must be able to see logs + paths: + - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + - rpm + - builddir/_CPack_Packages/Linux/RPM/SPECS/ + - dependencies.dot + - dependencies.png centos8: stage: build @@ -119,16 +191,16 @@ centos8: - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm - yum install -y ccache # From EPEL - source /etc/profile.d/ccache.sh - - export CCACHE_DIR="$(pwd)/.ccache"; ccache -s + - export CCACHE_DIR="$(pwd)/.ccache"; ccache --zero-stats # This repository does not have any .spec files, so install dependencies based on CentOS spec file - yum-builddep -y mariadb-server - mkdir builddir; cd builddir - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log - - eatmydata make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log - # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running - # multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968 - - make test || true # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820 - # - make test-force || true # mysql-test-runner takes too long, run it in a separate job instead + - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just + # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968 + - make test + # - make test-force # mysql-test-runner takes too long, run it MTR a separate job instead - *rpm_listfiles - mkdir ../rpm; mv *.rpm ../rpm - ccache -s @@ -140,7 +212,7 @@ centos8: - rpm - builddir/_CPack_Packages/Linux/RPM/SPECS/ cache: - key: $MARIADB_MAJOR_VERSION-$CI_JOB_NAME + key: $MARIADB_MAJOR_VERSION paths: - .ccache @@ -157,11 +229,11 @@ centos7: - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel - mkdir builddir; cd builddir - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log - - make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log - # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running - # multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968 - - make test || true # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820 - # - make test-force || true # mysql-test-runner takes too long, run it in a separate job instead + - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log + # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just + # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968 + - make test + # - make test-force # mysql-test-runner takes too long, run it in a separate job instead - *rpm_listfiles - mkdir ../rpm; mv *.rpm ../rpm artifacts: @@ -178,20 +250,20 @@ mysql-test-run: - fedora script: # Install packages so tests and the dependencies install - # @TODO: RPM missing 'patch' as dependency, so installing it manually for now - - yum install -y rpm/*.rpm patch + # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now + - yum install -y rpm/*.rpm patch diffutils # @TODO: Fix on packaging level for /usr/share/mariadb to work and errormsg.sys be found - rm -rf /usr/share/mariadb; ln -s /usr/share/mysql /usr/share/mariadb # mtr expects to be launched in-place and with write access to it's own directories - cd /usr/share/mysql-test + # Skip failing tests - | echo " - main.mysqldump : flaky on Gitlab-CI - main.flush_logs_not_windows : flaky in containers in general - main.mysql_upgrade_noengine : requires diff but diffutils is not a dependency + main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE' + main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004) + main.mysql_upgrade_noengine : upgrade output order does not match the expected " > skiplist - # @TODO: Flaky tests are skipped for now, but should be fixed - - ./mtr --suite=main --force --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist + - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist artifacts: when: always # Also show results when tests fail reports: @@ -226,11 +298,27 @@ fedora install: # Fedora does not support running services in Docker (like Debian packages do) so start it manually - /usr/bin/mariadb-install-db -u mysql - sudo -u mysql /usr/sbin/mariadbd & sleep 10 - # @TODO: Since we did a manual start, we also need to run upgrade manually - - /usr/bin/mariadb-upgrade -u root --socket /var/lib/mysql/mysql.sock + # Dump database contents as is before upgrade + - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > installed-database.sql + # Since we did a manual start, we also need to run upgrade manually + - /usr/bin/mariadb-upgrade -u root + # Dump database contents as is after upgrade + - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > upgraded-database.sql - | - mysql --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version - grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly" + mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version + grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly" + - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost" + - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS" + - mariadb -e "SHUTDOWN;" + - rm -rf /var/lib/mysql/* # Clear datadir before next run + # Start database without install-db step + - sudo -u mysql /usr/sbin/mariadbd --skip-network --skip-grant & sleep 10 + # Dump database contents in initial state + - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > empty-database.sql + artifacts: + paths: + - installed-database.sql + - upgraded-database.sql fedora upgrade: stage: test @@ -242,24 +330,39 @@ fedora upgrade: - /usr/libexec/mysql-check-socket - /usr/libexec/mysql-prepare-db-dir - sudo -u mysql /usr/libexec/mysqld --basedir=/usr & sleep 10 + # Dump database contents in installed state + - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-installed-database.sql - /usr/libexec/mysql-check-upgrade - - mysql --skip-column-names -e "SELECT @@version, @@version_comment" # Show version + # Dump database contents in upgraded state + - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-upgraded-database.sql + - mariadb --skip-column-names -e "SELECT @@version, @@version_comment" # Show version # @TODO: Upgrade from Fedora 33 MariaDB 10.4 to MariaDB.org latest does not work # so do this manual step to remove conflicts until packaging is fixed - - > - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check - mariadb-backup mariadb-connector-c-config + - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check mariadb-backup mariadb-connector-c-config - rm -f rpm/*debuginfo* # Not relevant in this test - - yum install -y rpm/*.rpm procps # procps provides pkill + - yum install -y rpm/*.rpm # nothing provides galera-4 on Fedora, so this step fails if built with wsrep - - pkill mysqld || true; sleep 5; pkill mysqld || true; sleep 5 - - /usr/bin/mariadb-install-db -u mysql + - mysql -e "SHUTDOWN;" + - /usr/bin/mariadb-install-db # This step should not do anything on upgrades, just exit - sudo -u mysql /usr/sbin/mariadbd & sleep 10 + # Dump database contents in installed state + - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-installed-database.sql || true + # The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577) # @TODO: Since we did a manual start, we also need to run upgrade manually - - /usr/bin/mariadb-upgrade -u root --socket /var/lib/mysql/mysql.sock + - /usr/bin/mariadb-upgrade + # Dump database contents in upgraded state + - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-upgraded-database.sql - | - mysql --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version + mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly" + - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost" + - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS" + artifacts: + paths: + - old-installed-database.sql + - old-upgraded-database.sql + - new-installed-database.sql + - new-upgraded-database.sql # Once all RPM builds and tests have passed, also run the DEB builds and tests # @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com -- cgit v1.2.1