Misc improvements to the Gitlab-CI pipeline for MariaDB

- Add new Ninja and Clang build jobs. This helps to ensure those
  toolchains also work in addition to default CMake/gcc.

- Generate dependencies.dot/png to illustrate the CMake/Make/Ninja
  build dependencies. Viewing this image and identifying bottle necks
  in parallelism can help make the build run faster.

- Enable CUnit tests now as they are fixed on 10.6 (MDEV-25820).

- Limit parallel builds to 2 CPUs (full parallelism needs MDEV-25968) on
  CMake/Make. Now only the Ninja builds run full parallel builds as only
  Ninja is smart enough to prevent builds failing on resource
  over-consumption.

- Enable Gitlab-CI cache for job 'centos8' for ccache so that it builds
  faster. Don't use Gitlab-CI cache for other jobs, as it would too easily
  use up all free tier storage on Gitlab.com and force users to get a paid
  account just for MariaDB builds.

- On other jobs clean away ccache, as it only had a 5% hit rate on single
  builds with no downloaded cache.

- Dump full database contents during the test install so that one can
  use diff to compare the database contents at different stages and thus
  track/debug potential bugs in mariadb-install-db and mariadb-upgrade
  code.

Bugfixes:

- Zero out ccache stats before each run so that 'ccache -s' would actually
  show the stats for the latest run.

1 files changed, 152 insertions, 49 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0a1bf5bff71..cd22fc5dd8c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,8 +37,22 @@ image: fedora:33
 # many components that are otherwise slow to build.
 variables:
   CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF"
-  # Major version dictates which branches share the same ccache
+  # Major version dictates which branches share the same ccache. E.g. 10.6-abc
+  # and 10.6-xyz will have the same cache.
   MARIADB_MAJOR_VERSION: "10.6"
+  # NOTE! Currently ccache is only used on the Centos8 build. As each job has
+  # sufficiently different environments they are unable to benefit from each
+  # other's ccaches. As each build generates about 1 GB of ccache, having
+  # multiple caches would quickly consume all free storage on Gitlab-CI and
+  # grind all builds to a halt. Also the network overhead of download/upload
+  # decreases the benefit of ccache in Gitlab-CI, and current cache:when and
+  # cache:policy are not flexible enough to have a system where the cache is
+  # uploaded only once a week and not on every build. Having ccache on at least
+  # one build still helps ensure that ccache compatibility is at least tested
+  # and if the Centos 8 build is always significantly faster than all other
+  # builds (e.g. on self-hosted Gitlab instances) then users would at least be
+  # able to discover it.
+  #
   # Most steps don't need the source code, only artifacts
   GIT_STRATEGY: none
   # Hack to satisfy directory name length requirement by CPackRPM in CMake 3.x
@@ -65,23 +79,21 @@ fedora:
     GIT_STRATEGY: fetch
     GIT_SUBMODULE_STRATEGY: normal
   script:
-    - yum install -y yum-utils rpm-build ccache openssl-devel
-    - source /etc/profile.d/ccache.sh
-    - export CCACHE_DIR="$(pwd)/.ccache"; ccache -s
+    - yum install -y yum-utils rpm-build openssl-devel graphviz
     # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
     - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
     # This repository does not have any .spec files, so install dependencies based on Fedora spec file
     - yum-builddep -y mariadb-server
     - mkdir builddir; cd builddir
     - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    - eatmydata make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running
-    # multi-proc, needs more debugging
-    - make test || true  # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820
-    # - make test-force || true  # mysql-test-runner takes too long, run it in a separate job instead
+    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run MTR in a separate job instead
     - *rpm_listfiles
     - mkdir ../rpm; mv *.rpm ../rpm
-    - ccache -s
   artifacts:
     when: always  # Must be able to see logs
     paths:
@@ -89,14 +101,74 @@ fedora:
       - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
       - rpm
       - builddir/_CPack_Packages/Linux/RPM/SPECS/
-  cache:
-    key: $MARIADB_MAJOR_VERSION-$CI_JOB_NAME
+      - dependencies.dot
+      - dependencies.png
+
+fedora-ninja:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel graphviz ninja-build
+    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - cmake -DRPM=generic $CMAKE_FLAGS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -G Ninja .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - ninja -t graph > ../dependencies.dot && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata ninja package --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # Ninja builds are not affected by bug https://jira.mariadb.org/browse/MDEV-25968
+    - ninja test
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
     paths:
-      - .ccache
-    # policy: pull
-    # @TODO: It would be enough to only download the cache. There is no need for
-    # every job to upload a new cache every time. A monthly or weekly scheduled
-    # run could update the cache for all other builds to us.
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+      - dependencies.dot
+      - dependencies.png
+
+fedora-clang:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel graphviz clang
+    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - export CXX=${CXX:-clang++}
+    - export CC=${CC:-clang}
+    - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+    - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+    - export CFLAGS='-Wno-unused-command-line-argument'
+    - export CXXFLAGS='-Wno-unused-command-line-argument'
+    - cmake -DRPM=generic $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run MTr in a separate job instead
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+      - dependencies.dot
+      - dependencies.png
 
 centos8:
   stage: build
@@ -119,16 +191,16 @@ centos8:
     - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
     - yum install -y ccache  # From EPEL
     - source /etc/profile.d/ccache.sh
-    - export CCACHE_DIR="$(pwd)/.ccache"; ccache -s
+    - export CCACHE_DIR="$(pwd)/.ccache"; ccache --zero-stats
     # This repository does not have any .spec files, so install dependencies based on CentOS spec file
     - yum-builddep -y mariadb-server
     - mkdir builddir; cd builddir
     - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    - eatmydata make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running
-    # multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
-    - make test || true  # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820
-    # - make test-force || true  # mysql-test-runner takes too long, run it in a separate job instead
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run it MTR a separate job instead
     - *rpm_listfiles
     - mkdir ../rpm; mv *.rpm ../rpm
     - ccache -s
@@ -140,7 +212,7 @@ centos8:
       - rpm
       - builddir/_CPack_Packages/Linux/RPM/SPECS/
   cache:
-    key: $MARIADB_MAJOR_VERSION-$CI_JOB_NAME
+    key: $MARIADB_MAJOR_VERSION
     paths:
       - .ccache
 
@@ -157,11 +229,11 @@ centos7:
     - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel
     - mkdir builddir; cd builddir
     - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    - make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running
-    # multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
-    - make test || true  # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820
-    # - make test-force || true  # mysql-test-runner takes too long, run it in a separate job instead
+    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run it in a separate job instead
     - *rpm_listfiles
     - mkdir ../rpm; mv *.rpm ../rpm
   artifacts:
@@ -178,20 +250,20 @@ mysql-test-run:
     - fedora
   script:
     # Install packages so tests and the dependencies install
-    # @TODO: RPM missing 'patch' as dependency, so installing it manually for now
-    - yum install -y rpm/*.rpm patch
+    # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
+    - yum install -y rpm/*.rpm patch diffutils
     # @TODO: Fix on packaging level for /usr/share/mariadb to work and errormsg.sys be found
     - rm -rf /usr/share/mariadb; ln -s /usr/share/mysql /usr/share/mariadb
     # mtr expects to be launched in-place and with write access to it's own directories
     - cd /usr/share/mysql-test
+    # Skip failing tests
     - |
       echo "
-      main.mysqldump : flaky on Gitlab-CI
-      main.flush_logs_not_windows : flaky in containers in general
-      main.mysql_upgrade_noengine : requires diff but diffutils is not a dependency
+      main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
+      main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
+      main.mysql_upgrade_noengine : upgrade output order does not match the expected
       " > skiplist
-    # @TODO: Flaky tests are skipped for now, but should be fixed
-    - ./mtr --suite=main --force --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
+    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
   artifacts:
     when: always  # Also show results when tests fail
     reports:
@@ -226,11 +298,27 @@ fedora install:
     # Fedora does not support running services in Docker (like Debian packages do) so start it manually
     - /usr/bin/mariadb-install-db -u mysql
     - sudo -u mysql /usr/sbin/mariadbd & sleep 10
-    # @TODO: Since we did a manual start, we also need to run upgrade manually
-    - /usr/bin/mariadb-upgrade -u root --socket /var/lib/mysql/mysql.sock
+    # Dump database contents as is before upgrade
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > installed-database.sql
+    # Since we did a manual start, we also need to run upgrade manually
+    - /usr/bin/mariadb-upgrade -u root
+    # Dump database contents as is after upgrade
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > upgraded-database.sql
     - |
-      mysql --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
-      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
+      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
+    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+    - mariadb -e "SHUTDOWN;"
+    - rm -rf /var/lib/mysql/*  # Clear datadir before next run
+    # Start database without install-db step
+    - sudo -u mysql /usr/sbin/mariadbd --skip-network --skip-grant & sleep 10
+    # Dump database contents in initial state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > empty-database.sql
+  artifacts:
+    paths:
+      - installed-database.sql
+      - upgraded-database.sql
 
 fedora upgrade:
   stage: test
@@ -242,24 +330,39 @@ fedora upgrade:
     - /usr/libexec/mysql-check-socket
     - /usr/libexec/mysql-prepare-db-dir
     - sudo -u mysql /usr/libexec/mysqld --basedir=/usr & sleep 10
+    # Dump database contents in installed state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-installed-database.sql
     - /usr/libexec/mysql-check-upgrade
-    - mysql --skip-column-names -e "SELECT @@version, @@version_comment"  # Show version
+    # Dump database contents in upgraded state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-upgraded-database.sql
+    - mariadb --skip-column-names -e "SELECT @@version, @@version_comment"  # Show version
     # @TODO: Upgrade from Fedora 33 MariaDB 10.4 to MariaDB.org latest does not work
     # so do this manual step to remove conflicts until packaging is fixed
-    - >
-      yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check
-      mariadb-backup mariadb-connector-c-config
+    - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check mariadb-backup mariadb-connector-c-config
     - rm -f rpm/*debuginfo*  # Not relevant in this test
-    - yum install -y rpm/*.rpm procps  # procps provides pkill
+    - yum install -y rpm/*.rpm
     # nothing provides galera-4 on Fedora, so this step fails if built with wsrep
-    - pkill mysqld || true; sleep 5; pkill mysqld || true; sleep 5
-    - /usr/bin/mariadb-install-db -u mysql
+    - mysql -e "SHUTDOWN;"
+    - /usr/bin/mariadb-install-db  # This step should not do anything on upgrades, just exit
     - sudo -u mysql /usr/sbin/mariadbd & sleep 10
+    # Dump database contents in installed state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-installed-database.sql || true
+    # The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577)
     # @TODO: Since we did a manual start, we also need to run upgrade manually
-    - /usr/bin/mariadb-upgrade -u root --socket /var/lib/mysql/mysql.sock
+    - /usr/bin/mariadb-upgrade
+    # Dump database contents in upgraded state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-upgraded-database.sql
     - |
-      mysql --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
       grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
+    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+  artifacts:
+    paths:
+      - old-installed-database.sql
+      - old-upgraded-database.sql
+      - new-installed-database.sql
+      - new-upgraded-database.sql
 
 # Once all RPM builds and tests have passed, also run the DEB builds and tests
 # @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com