diff options
author | Otto Kekäläinen <otto@kekalainen.net> | 2021-10-11 12:53:11 -0700 |
---|---|---|
committer | Otto Kekäläinen <otto@kekalainen.net> | 2022-01-22 13:47:39 -0800 |
commit | c5c61b51b66df66dbd49a88111ed7f57705dcc78 (patch) | |
tree | ad985a60850ab6bacd54d387445506fd1c09301c | |
parent | 1f5fc7b745fd5009325fcf94be67c3f850408266 (diff) | |
download | mariadb-git-c5c61b51b66df66dbd49a88111ed7f57705dcc78.tar.gz |
Extend the Gitlab-CI pipeline to run mini benchmark
Implement new mini-benchmark script for simple CPU bound benchmark for the
duration of 5 minutes. The script can be run stand-alone or as part of a
CI pipeline.
Extend Gitlab-CI to run mini-benchmark on every commit to catch if there
are severe performance regressions.
Also bump MARIADB_MAJOR_VERSION to 10.8 which is needed on the 10.8 branch.
-rw-r--r-- | .gitlab-ci.yml | 31 | ||||
-rw-r--r-- | debian/mariadb-server-10.8.install | 1 | ||||
-rw-r--r-- | support-files/CMakeLists.txt | 4 | ||||
-rwxr-xr-x | support-files/mini-benchmark.sh | 229 |
4 files changed, 262 insertions, 3 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd22fc5dd8c..003278fa31c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ variables: CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF" # Major version dictates which branches share the same ccache. E.g. 10.6-abc # and 10.6-xyz will have the same cache. - MARIADB_MAJOR_VERSION: "10.6" + MARIADB_MAJOR_VERSION: "10.8" # NOTE! Currently ccache is only used on the Centos8 build. As each job has # sufficiently different environments they are unable to benefit from each # other's ccaches. As each build generates about 1 GB of ccache, having @@ -364,6 +364,35 @@ fedora upgrade: - new-installed-database.sql - new-upgraded-database.sql +mini-benchmark: + stage: test + dependencies: + - fedora + script: + - ls -la rpm; rm -vf rpm/*.el?.* # Delete artifacts from Centos builds + # Don't use cracklib, otherwise the Sysbench user password will be rejected + - rm -vf rpm/*cracklib*.rpm + # Nothing provides galera-4 on Fedora, so this step fails if built with wsrep + - yum install -y rpm/*.rpm + # Fedora does not support running services in Docker (like Debian packages do) so start it manually + - /usr/bin/mariadb-install-db -u mysql + - sudo -u mysql /usr/sbin/mariadbd & sleep 10 + # Since we did a manual start, we also need to run upgrade manually + - /usr/bin/mariadb-upgrade -u root + - | + mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version + grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly" + - yum install -y sysbench procps-ng perf || yum install -y https://kojipkgs.fedoraproject.org//packages/luajit/2.0.4/3.el7/x86_64/luajit-2.0.4-3.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/sysbench/1.0.17/2.el7/x86_64/sysbench-1.0.17-2.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/ck/0.5.2/2.el7/x86_64/ck-0.5.2-2.el7.x86_64.rpm + - /usr/share/mysql/mini-benchmark + - cp -av */sysbench-run-*.log */metrics.txt .. # Move files one level down so they can be saved as artifacts + artifacts: + when: always + paths: + - sysbench-run-*.log + reports: + metrics: + - metrics.txt + # Once all RPM builds and tests have passed, also run the DEB builds and tests # @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com # runners are too small for everything this stage does. diff --git a/debian/mariadb-server-10.8.install b/debian/mariadb-server-10.8.install index e1a05261ef2..71fe6bc77d5 100644 --- a/debian/mariadb-server-10.8.install +++ b/debian/mariadb-server-10.8.install @@ -78,5 +78,6 @@ usr/share/man/man1/wsrep_sst_mysqldump.1 usr/share/man/man1/wsrep_sst_rsync.1 usr/share/man/man1/wsrep_sst_rsync_wan.1 usr/share/mysql/errmsg-utf8.txt +usr/share/mysql/mini-benchmark usr/share/mysql/wsrep.cnf usr/share/mysql/wsrep_notify diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index 90e79954fd1..09b8e545179 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -50,8 +50,8 @@ ENDIF() IF(UNIX AND NOT WITHOUT_SERVER) SET(prefix ${CMAKE_INSTALL_PREFIX}) - FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure wsrep_notify) - CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${script}.sh + FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure wsrep_notify mini-benchmark) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${script}.sh ${CMAKE_CURRENT_BINARY_DIR}/${script} @ONLY ) INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${script} DESTINATION ${inst_location} COMPONENT Server_Scripts) diff --git a/support-files/mini-benchmark.sh b/support-files/mini-benchmark.sh new file mode 100755 index 00000000000..d5dd194bc3f --- /dev/null +++ b/support-files/mini-benchmark.sh @@ -0,0 +1,229 @@ +#!/bin/bash +# Abort on errors +set -e + +display_help() { + echo "Usage: $(basename "$0") [-h] [--perf] [--perf-flamegraph]" + echo + echo "This is a very small and naive benchmark script designed to be suitable" + echo "for running in a CI system on every commit to detect severe performance" + echo "regressions." + echo + echo "optional arguments:" + echo " --perf measure CPU cycles and instruction count in for " + echo " sysbench runs" + echo " --perf-flamegraph record performance counters in perf.data.* and" + echo " generate flamegraphs automatically" + echo " -h, --help display this help and exit" +} + +while : +do + case "$1" in + -h | --help) + display_help + exit 0 + ;; + --version) + display_version + exit 0 + ;; + --perf) + PERF=true + shift + ;; + --perf-flamegraph) + PERF_RECORD=true + shift + ;; + -*) + echo "Error: Unknown option: $1" >&2 + ## or call function display_help + exit 1 + ;; + *) # No more options + break + ;; + esac +done + +# Check that the dependencies of this script are available +if [ ! -e /usr/bin/pgrep ] +then + echo "ERROR: Command 'pgrep' missing, please install package 'psproc'" + exit 1 +fi + +if [ ! -e /usr/bin/sysbench ] +then + echo "ERROR: Command 'sysbench' missing, please install package 'sysbench'" + exit 1 +fi + +# If there are multiple processes, assume the last one is the actual server and +# any potential other ones were just part of the service wrapper chain +MARIADB_SERVER_PID="$(echo "$(pgrep -f mariadbd || pgrep -f mysqld)" | tail -n 1)" + +if [ -z "$MARIADB_SERVER_PID" ] +then + echo "ERROR: Server 'mariadbd' or 'mysqld' is not running, please start the service" + exit 1 +fi + +if [ "$PERF" == true ] || [ "$PERF_RECORD" == true ] +then + if [ ! -e /usr/bin/perf ] + then + echo "ERROR: Command 'perf' missing, please install package 'perf'" + exit 1 + fi +fi + +if [ "$PERF_RECORD" == true ] +then + if [ ! -e /usr/bin/flamegraph.pl ] + then + echo "ERROR: Command 'flamegraph.pl' missing, please install package 'flamegraph'" + exit 1 + fi + + if [ ! -e /usr/bin/stackcollapse-perf.pl ] + then + echo "ERROR: Command 'stackcollapse-perf.pl' missing, please install package 'flamegraph-stackcollapse-perf'" + exit 1 + fi + + if [ ! -e /usr/bin/debuginfo-install ] + then + echo "ERROR: Command 'debuginfo-install' missing, please install package 'dnf-utils'" + exit 1 + fi + + echo "Ensure the MariaDB Server debug symbols are installed" + for x in $(ldd /usr/sbin/mariadbd | grep -oE " /.* ") + do + rpm -q --whatprovides --qf '%{name}' $x | cut -d : -f 1 + done | sort -u > mariadbd-dependencies.txt + # shellcheck disable=SC2046 + debuginfo-install -y mariadb-server $(cat mariadbd-dependencies.txt) + + echo "Using 'perf' to record performance counters in perf.data files" + PERF="perf record -g --freq=99 --output=perf.data --timestamp-filename --pid=$MARIADB_SERVER_PID --" + +elif [ -e /usr/bin/perf ] +then + # If flamegraphs were not requested, log normal perf counters if possible + echo "Using 'perf' to log basic performance counters for benchmark" +fi + PERF="perf stat -p $MARIADB_SERVER_PID --" + +# Run sysbench on another CPU if system has more than one available +if [ "$(nproc)" -gt 1 ] +then + TASKSET_SYSBENCH='taskset -c 1' +else + TASKSET_SYSBENCH='' +fi + +echo "System hardware information:" +lscpu +free -m +df -h . +uname -a +echo + +echo "Set highest priority for MariaDB Server process ID $MARIADB_SERVER_PID" +renice --priority -20 --pid "$MARIADB_SERVER_PID" + +echo "Set CPU affinity 0 for MariaDB Server process ID $MARIADB_SERVER_PID" +taskset -cp 0 "$MARIADB_SERVER_PID" + +mariadb -e " + CREATE DATABASE IF NOT EXISTS sbtest; + CREATE USER IF NOT EXISTS sbtest@localhost; + GRANT ALL PRIVILEGES ON sbtest.* TO sbtest@localhost" + +sysbench oltp_read_write prepare --tables=20 --table-size=100000 | tee sysbench-prepare.log +sync && sleep 1 # Ensure writes were propagated to disk + +# Save results of this run in a subdirectory so that they are not overwritten by +# the next run +TIMESTAMP="$(date -Iseconds)" +mkdir "mini-benchmark-$TIMESTAMP" +cd "mini-benchmark-$TIMESTAMP" || exit 1 + +# Run benchmark with increasing thread counts. The MariaDB Server will be using +# around 300 MB of RAM and mostly reading and writing in RAM, so I/O usage is +# also low. The benchmark will most likely be CPU bound to due to the load +# profile, and also guaranteed to be CPU bound because of being limited to a +# single CPU with 'tasksel'. +for t in 1 2 4 8 16 +do + # Prepend command with perf if defined + # Output stderr to stdout as perf outpus everything in stderr + $PERF $TASKSET_SYSBENCH sysbench oltp_read_write run --threads=$t --time=60 --report-interval=10 2>&1 | tee sysbench-run-$t.log +done + +sysbench oltp_read_write cleanup --tables=20 | tee sysbench-cleanup.log + +# Store results from 4 thread run in a Gitlab-CI compatible metrics file +grep -oE '[a-z]+:[ ]+[0-9.]+' sysbench-run-4.log | sed -r 's/\s+/ /g' | tail -n 15 > metrics.txt + +echo # Newline improves readability +echo "== SUMMARY ==" + +# Print performance counter summary if they were logged +if grep --quiet cycles sysbench-run-*.log +then + grep -e cycles sysbench-run-*.log | sort -k 2 + echo "Total: $(grep -h -e cycles sysbench-run-*.log | sort -k 1 | awk '{s+=$1}END{print s}')" + echo # Newline improves readability + grep -e instructions sysbench-run-*.log | sort -k 2 + echo "Total: $(grep -h -e instructions sysbench-run-*.log | sort -k 1 | awk '{s+=$1}END{print s}')" + echo # Newline improves readability + + # Final verdict based on cpu cycle count + RESULT="$(grep -h -e cycles sysbench-run-*.log | sort -k 1 | awk '{s+=$1}END{print s}')" + if [ "$RESULT" -gt 850000000000 ] + then + echo # Newline improves readability + echo "Benchmark exceeded 8.5 billion cpu cycles, performance most likely regressed!" + exit 1 + fi +fi + +# List all sysbench status lines at once +grep -h thds sysbench-run-*.log | sort -k 5 -h + +echo # Newline improves readability +echo "Highest count for queries per second:" +sort -k 9 -h sysbench-run-*.log | tail -n 1 + +if [ "$PERF_RECORD" == true ] +then + for f in perf.data.* + do + perf script -i $f | stackcollapse-perf.pl | flamegraph.pl --width 3000 > $f.svg + done + echo "Flamegraphs stored in folder mini-benchmark-$TIMESTAMP/" +fi + +# Fallback if CPU cycle count not availalbe: final verdict based on peak QPS +RESULT="$(sort -k 9 -h sysbench-run-*.log | tail -n 1 | grep -oE "qps: [0-9]+" | grep -oE "[0-9]+")" +case $RESULT in + ''|*[!0-9]*) + echo "ERROR: Benchmark result invalid, not an integer." + exit 1 + ;; + + *) + if [ "$RESULT" -lt 13000 ] + then + echo # Newline improves readability + echo "Benchmark did not reach 13000+ qps, performance most likely regressed!" + exit 1 + else + echo "Banchmark passed with $RESULT queries per second as peak value" + fi + ;; +esac |