Backport GitLab CI to earlier branches

Add .gitlab-ci.yml file to earliest supported branch to enable
automated building and testing for all MariaDB major branches.

For 10.4 we use the bundled SSL to build MariaDB when the platform
does not have OpenSSL 1.1 available. This requires the installation of
gnutls-devel as a dependency of MariaDB Connector/C.
OpenSSL 3.0 support was backported to MariaDB 10.5
(see https://github.com/MariaDB/server/pull/2036, f0fa40ef, 8a9c1e9c)

All new code of the whole pull request, including one or several files
that are either new files or modified ones, are contributed under the
BSD-new license. I am contributing on behalf of my employer
Amazon Web Services, Inc.

1 files changed, 494 insertions, 0 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000000..39dae0facb8
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,494 @@
+---
+# This Gitlab-CI pipeline offers basic validation that a commit did not
+# introduce easily detectable regressions. Builds run primairly on a new Fedora,
+# which has all the latest upstream build dependencies and thus is the primary
+# testing target, as eventually everything in Fedora becomes the next CentOS and
+# Red Hat releases.
+#
+# In addition test building on CentOS 7 and 8 to ensure that the code base
+# remains reasonably backwards compatible.
+#
+# This is now intentionally simple, to keep it fast and accurate with minimal
+# false positive failures. If one wants to extend it, see debian/salsa-ci.yml
+# for inspiration on more integration tests to run.
+#
+# Also make sure the pipeline stays within the bounds of what CI workers on
+# Gitlab-CI are capable of executing, thus ensuring that any potential
+# contributor can at any point in time fork to their own Gitlab account and
+# start working towards meaningful contributions!
+#
+# NOTE TO MERGERS: Most of the contents in the Gitlab-CI configuration has been
+# tailored for a specific release or MariaDB. As a general rule, do not merge
+# changes in this file across MariaDB branches to avoid breaking the CI. Updates
+# the Gitlab-CI pipeline are most of the time better done manually per major
+# release branch.
+
+stages:
+  - build
+  - test
+  - Salsa-CI
+
+default:
+  # Base image for builds and tests unless otherwise defined
+  image: fedora:latest
+  # Extend build jobs to have longer timeout as the default GitLab
+  # timeout (1h) is often not enough
+  timeout: 3h
+
+# Define common CMAKE_FLAGS for all builds. Skim down build by omitting all
+# submodules (a commit in this repo does not affect their builds anyway) and
+# many components that are otherwise slow to build.
+variables:
+  CMAKE_FLAGS: "-DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF"
+  # Major version dictates which branches share the same ccache. E.g. 10.6-abc
+  # and 10.6-xyz will have the same cache.
+  MARIADB_MAJOR_VERSION: "10.6"
+  # NOTE! Currently ccache is only used on the Centos8 build. As each job has
+  # sufficiently different environments they are unable to benefit from each
+  # other's ccaches. As each build generates about 1 GB of ccache, having
+  # multiple caches would quickly consume all free storage on Gitlab-CI and
+  # grind all builds to a halt. Also the network overhead of download/upload
+  # decreases the benefit of ccache in Gitlab-CI, and current cache:when and
+  # cache:policy are not flexible enough to have a system where the cache is
+  # uploaded only once a week and not on every build. Having ccache on at least
+  # one build still helps ensure that ccache compatibility is at least tested
+  # and if the Centos 8 build is always significantly faster than all other
+  # builds (e.g. on self-hosted Gitlab instances) then users would at least be
+  # able to discover it.
+  #
+  # Most steps don't need the source code, only artifacts
+  GIT_STRATEGY: none
+  # Hack to satisfy directory name length requirement by CPackRPM in CMake 3.x
+  # https://cmake.org/cmake/help/v3.7/module/CPackRPM.html#variable:CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX
+  GIT_CLONE_PATH: $CI_BUILDS_DIR/CPACK_BUILD_SOURCE_DIRS_LONG_NAME_REQUIREMENT
+
+# Define once, use many times
+.rpm_listfiles: &rpm_listfiles
+  - |
+    echo "Generating rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log ..."
+    for package in *.rpm
+    do
+      echo "$package"
+      rpm -qlpv "$package" | awk '{print $1 " " $3 "/" $4 " ." $9 " " $10 " " $11}' | sort -k 3
+      echo "------------------------------------------------"
+    done >> "../rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log"
+  # CPackRPM lists contents in build log, so no need to show the output of this,
+  # just store it as a build artifact that can be downloaded and diffed against
+  # other builds to detect which files where added/removed/moved
+
+fedora:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel graphviz clang gnutls-devel
+    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS -DWITH_SSL=bundled .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run MTR in a separate job instead
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+      - dependencies.dot
+      - dependencies.png
+
+fedora-ninja:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel graphviz ninja-build gnutls-devel
+    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - cmake -DRPM=generic $CMAKE_FLAGS -DWITH_SSL=bundled -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -G Ninja .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - ninja -t graph > ../dependencies.dot && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata ninja package -j 2 --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Unlike other builds, the Ninja builds using Gitlab.com runners don't get stuck, but they do get
+    # stuck on runners with more processors, see https://jira.mariadb.org/browse/MDEV-25968.
+    # Thus, use the same limitation on Ninja builds as well to ensure it never gets stuck due to this bug.
+    - ninja test
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+      - dependencies.dot
+      - dependencies.png
+
+fedora-clang:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel graphviz clang gnutls-devel
+    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - export CXX=${CXX:-clang++}
+    - export CC=${CC:-clang}
+    - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+    - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+    - export CFLAGS='-Wno-unused-command-line-argument'
+    - export CXXFLAGS='-Wno-unused-command-line-argument'
+    - cmake -DRPM=generic $CMAKE_FLAGS -DWITH_SSL=bundled .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run MTr in a separate job instead
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+      - dependencies.dot
+      - dependencies.png
+
+fedora-sanitizer:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel clang gnutls-devel
+    - yum install -y libasan libtsan libubsan
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - export CXX=${CXX:-clang++}
+    - export CC=${CC:-clang}
+    - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+    - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+    - export CFLAGS='-Wno-unused-command-line-argument'
+    - export CXXFLAGS='-Wno-unused-command-line-argument'
+    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS -DWITH_SSL=bundled $SANITIZER .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: the build will fail consistently at 24% when trying to make using eatmydata
+    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+  parallel:
+    matrix:
+      - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES, -DWITH_MSAN=YES]
+
+centos8:
+  stage: build
+  image: quay.io/centos/centos:stream8 # CentOS 8 is deprecated, use this Stream8 instead
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel pcre2-devel
+    - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+    # dnf --enablerepo=powertools install Judy-devel  #--> not found
+    - dnf config-manager --set-enabled powertools
+    # Error:
+    # Problem: conflicting requests
+    # - package Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.i686 is filtered out by modular filtering
+    # - package Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.x86_64 is filtered out by modular filtering
+    # Solution: install Judy-devel directly from downloaded rpm file:
+    - yum install -y http://vault.centos.org/centos/8/PowerTools/x86_64/os/Packages/Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.x86_64.rpm
+    # Use eatmydata to speed up build
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    - yum install -y ccache  # From EPEL
+    - source /etc/profile.d/ccache.sh
+    - export CCACHE_DIR="$(pwd)/.ccache"; ccache --zero-stats
+    # This repository does not have any .spec files, so install dependencies based on CentOS spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS -DWITH_SSL=system .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run it MTR a separate job instead
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+    - ccache -s
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+  cache:
+    key: $MARIADB_MAJOR_VERSION
+    paths:
+      - .ccache
+
+centos7:
+  stage: build
+  image: centos:7
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    # ..with a few extra ones, as CentOS 7 is very old and these are added in newer MariaDB releases
+    - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel pcre2-devel
+    - mkdir builddir; cd builddir
+    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS -DWITH_SSL=system .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run it in a separate job instead
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+
+.mysql-test-run: &mysql-test-run-def
+  stage: test
+  script:
+    # Install packages so tests and the dependencies install
+    # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
+    - yum install -y rpm/*.rpm patch diffutils
+    # @TODO: Fix on packaging level for /usr/share/mariadb to work and errormsg.sys be found
+    - rm -rf /usr/share/mariadb; ln -s /usr/share/mysql /usr/share/mariadb
+    # mtr expects to be launched in-place and with write access to it's own directories
+    - cd /usr/share/mysql-test
+    # Skip failing tests
+    - |
+      echo "
+      main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
+      main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
+      main.mysql_upgrade_noengine : upgrade output order does not match the expected
+      " > skiplist
+    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
+
+mysql-test-run:
+  stage: test
+  dependencies:
+    - fedora
+  needs:
+    - fedora
+  <<: *mysql-test-run-def
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+# Duplicate of the above jobs, except we use sanitizer build jobs as a dependency. This is so we can keep
+# sanitizer errors separate from functional test failures. Currently, there is no way to run the same
+# job for different dependencies.
+#
+# Additionally, for each sanitizer MTR job, we enable --force-restart so that 
+# sanitizer errors can be traced to individual tests. The difference in test 
+# suite runtime as a result of this flag is negligable (~30s for the entire test suite).
+# (see https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_MYSQL_TEST_RUN_PL.html)
+mysql-test-run-asan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_ASAN=YES]"
+  needs:
+    - "fedora-sanitizer: [-DWITH_ASAN=YES]"
+  <<: *mysql-test-run-def
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-tsan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_TSAN=YES]"
+  needs:
+    - "fedora-sanitizer: [-DWITH_TSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-ubsan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
+  needs:
+    - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-msan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_MSAN=YES]"
+  needs:
+    - "fedora-sanitizer: [-DWITH_MSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+rpmlint:
+  stage: test
+  dependencies:
+    - fedora
+  needs:
+    - fedora
+  script:
+    - yum install -y rpmlint
+    - rm -f rpm/*debuginfo*  # Not relevant in this test
+    # Limit output to 1000 lines as Gitlab-CI max output is 4194304 bytes
+    # Save everything in a log file so it can be viewed in full via artifacts
+    - rpmlint --info rpm/*.rpm | tee -a rpmlint-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+  artifacts:
+    when: always  # Also show results when tests fail
+    paths:
+      - rpmlint-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+  allow_failure: true
+  # @TODO: The package is not rpmlint clean, must allow failure for now
+
+fedora install:
+  stage: test
+  dependencies:
+    - fedora
+  needs:
+    - fedora
+  script:
+    - rm -f rpm/*debuginfo*  # Not relevant in this test
+    # Nothing provides galera-4 on Fedora, so this step fails if built with wsrep
+    - yum install -y rpm/*.rpm
+    # Fedora does not support running services in Docker (like Debian packages do) so start it manually
+    - /usr/bin/mariadb-install-db -u mysql
+    - sudo -u mysql /usr/sbin/mariadbd & sleep 10
+    # Dump database contents as is before upgrade
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > installed-database.sql
+    # Since we did a manual start, we also need to run upgrade manually
+    - /usr/bin/mariadb-upgrade -u root
+    # Dump database contents as is after upgrade
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > upgraded-database.sql
+    - |
+      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
+    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+    - mariadb -e "SHUTDOWN;"
+    - rm -rf /var/lib/mysql/*  # Clear datadir before next run
+    # Start database without install-db step
+    - sudo -u mysql /usr/sbin/mariadbd --skip-network --skip-grant & sleep 10
+    # Dump database contents in initial state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > empty-database.sql
+  artifacts:
+    paths:
+      - installed-database.sql
+      - upgraded-database.sql
+
+fedora upgrade:
+  stage: test
+  dependencies:
+    - fedora
+  needs:
+    - fedora
+  script:
+    - dnf install -y mariadb-server
+    # Fedora does not support running services in Docker (like Debian packages do) so start it manually
+    - /usr/libexec/mariadb-check-socket
+    - /usr/libexec/mariadb-prepare-db-dir
+    - sudo -u mysql /usr/libexec/mariadbd --basedir=/usr & sleep 10
+    # Dump database contents in installed state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-installed-database.sql
+    - /usr/libexec/mariadb-check-upgrade
+    # Dump database contents in upgraded state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-upgraded-database.sql
+    - mariadb --skip-column-names -e "SELECT @@version, @@version_comment"  # Show version
+    # @TODO: Upgrade from Fedora 33 MariaDB 10.4 to MariaDB.org latest does not work
+    # so do this manual step to remove conflicts until packaging is fixed
+    - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check mariadb-backup mariadb-connector-c-config
+    - rm -f rpm/*debuginfo*  # Not relevant in this test
+    - yum install -y rpm/*.rpm
+    # nothing provides galera-4 on Fedora, so this step fails if built with wsrep
+    - mysql -e "SHUTDOWN;"
+    - /usr/bin/mariadb-install-db  # This step should not do anything on upgrades, just exit
+    - sudo -u mysql /usr/sbin/mariadbd & sleep 10
+    # Dump database contents in installed state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-installed-database.sql || true
+    # The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577)
+    # @TODO: Since we did a manual start, we also need to run upgrade manually
+    - /usr/bin/mariadb-upgrade
+    # Dump database contents in upgraded state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-upgraded-database.sql
+    - |
+      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
+    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+  artifacts:
+    paths:
+      - old-installed-database.sql
+      - old-upgraded-database.sql
+      - new-installed-database.sql
+      - new-upgraded-database.sql
+
+# Once all RPM builds and tests have passed, also run the DEB builds and tests
+# @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com
+# runners are too small for everything this stage does.
+# build_deb:
+#   stage: Salsa-CI
+#   trigger:
+#     include: debian/salsa-ci.yml