summaryrefslogtreecommitdiff
path: root/.gitlab-ci.yml
blob: 6db89cd332f3c25a78aa5e2da7e450f9317df3a0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
---
# This Gitlab-CI pipeline offers basic validation that a commit did not
# introduce easily detectable regressions. Builds run primairly on a new Fedora,
# which has all the latest upstream build dependencies and thus is the primary
# testing target, as eventually everything in Fedora becomes the next CentOS and
# Red Hat releases.
#
# In addition test building on CentOS 7 and 8 to ensure that the code base
# remains reasonably backwards compatible.
#
# This is now intentionally simple, to keep it fast and accurate with minimal
# false positive failures. If one wants to extend it, see debian/salsa-ci.yml
# for inspiration on more integration tests to run.
#
# Also make sure the pipeline stays within the bounds of what CI workers on
# Gitlab-CI are capable of executing, thus ensuring that any potential
# contributor can at any point in time fork to their own Gitlab account and
# start working towards meaningful contributions!
#
# NOTE TO MERGERS: Most of the contents in the Gitlab-CI configuration has been
# tailored for a specific release or MariaDB. As a general rule, do not merge
# changes in this file across MariaDB branches to avoid breaking the CI. Updates
# the Gitlab-CI pipeline are most of the time better done manually per major
# release branch.

stages:
  - build
  - test
  - Salsa-CI

# Base image for builds and tests unless otherwise defined
# @TODO: Fedora 34 is latest, but fails to start on Gitlab.com with error "shell not found"
image: fedora:33

# Define common CMAKE_FLAGS for all builds. Skim down build by omitting all
# submodules (a commit in this repo does not affect their builds anyway) and
# many components that are otherwise slow to build.
variables:
  CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF"
  # Major version dictates which branches share the same ccache. E.g. 10.6-abc
  # and 10.6-xyz will have the same cache.
  MARIADB_MAJOR_VERSION: "10.6"
  # NOTE! Currently ccache is only used on the Centos8 build. As each job has
  # sufficiently different environments they are unable to benefit from each
  # other's ccaches. As each build generates about 1 GB of ccache, having
  # multiple caches would quickly consume all free storage on Gitlab-CI and
  # grind all builds to a halt. Also the network overhead of download/upload
  # decreases the benefit of ccache in Gitlab-CI, and current cache:when and
  # cache:policy are not flexible enough to have a system where the cache is
  # uploaded only once a week and not on every build. Having ccache on at least
  # one build still helps ensure that ccache compatibility is at least tested
  # and if the Centos 8 build is always significantly faster than all other
  # builds (e.g. on self-hosted Gitlab instances) then users would at least be
  # able to discover it.
  #
  # Most steps don't need the source code, only artifacts
  GIT_STRATEGY: none
  # Hack to satisfy directory name length requirement by CPackRPM in CMake 3.x
  # https://cmake.org/cmake/help/v3.7/module/CPackRPM.html#variable:CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX
  GIT_CLONE_PATH: $CI_BUILDS_DIR/CPACK_BUILD_SOURCE_DIRS_LONG_NAME_REQUIREMENT

# Define once, use many times
.rpm_listfiles: &rpm_listfiles
  - |
    echo "Generating rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log ..."
    for package in *.rpm
    do
      echo "$package"
      rpm -qlpv "$package" | awk '{print $1 " " $3 "/" $4 " ." $9 " " $10 " " $11}' | sort -k 3
      echo "------------------------------------------------"
    done >> ../rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
  # CPackRPM lists contents in build log, so no need to show the output of this,
  # just store it as a build artifact that can be downloaded and diffed against
  # other builds to detect which files where added/removed/moved

fedora:
  stage: build
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: normal
  script:
    - yum install -y yum-utils rpm-build openssl-devel graphviz
    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
    - yum-builddep -y mariadb-server
    - mkdir builddir; cd builddir
    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
    - make test
    # - make test-force  # mysql-test-runner takes too long, run MTR in a separate job instead
    - *rpm_listfiles
    - mkdir ../rpm; mv *.rpm ../rpm
  artifacts:
    when: always  # Must be able to see logs
    paths:
      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpm
      - builddir/_CPack_Packages/Linux/RPM/SPECS/
      - dependencies.dot
      - dependencies.png

fedora-ninja:
  stage: build
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: normal
  script:
    - yum install -y yum-utils rpm-build openssl-devel graphviz ninja-build
    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
    - yum-builddep -y mariadb-server
    - mkdir builddir; cd builddir
    - cmake -DRPM=generic $CMAKE_FLAGS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -G Ninja .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    - ninja -t graph > ../dependencies.dot && dot -Tpng -o ../dependencies.png ../dependencies.dot
    - eatmydata ninja package --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    # Ninja builds are not affected by bug https://jira.mariadb.org/browse/MDEV-25968
    - ninja test
    - *rpm_listfiles
    - mkdir ../rpm; mv *.rpm ../rpm
  artifacts:
    when: always  # Must be able to see logs
    paths:
      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpm
      - builddir/_CPack_Packages/Linux/RPM/SPECS/
      - dependencies.dot
      - dependencies.png

fedora-clang:
  stage: build
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: normal
  script:
    - yum install -y yum-utils rpm-build openssl-devel graphviz clang
    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
    - yum-builddep -y mariadb-server
    - mkdir builddir; cd builddir
    - export CXX=${CXX:-clang++}
    - export CC=${CC:-clang}
    - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
    - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
    - export CFLAGS='-Wno-unused-command-line-argument'
    - export CXXFLAGS='-Wno-unused-command-line-argument'
    - cmake -DRPM=generic $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
    - make test
    # - make test-force  # mysql-test-runner takes too long, run MTr in a separate job instead
    - *rpm_listfiles
    - mkdir ../rpm; mv *.rpm ../rpm
  artifacts:
    when: always  # Must be able to see logs
    paths:
      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpm
      - builddir/_CPack_Packages/Linux/RPM/SPECS/
      - dependencies.dot
      - dependencies.png

centos8:
  stage: build
  image: quay.io/centos/centos:stream8 # CentOS 8 is deprecated, use this Stream8 instead
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: normal
  script:
    - yum install -y yum-utils rpm-build openssl-devel pcre2-devel
    - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
    # dnf --enablerepo=powertools install Judy-devel  #--> not found
    - dnf config-manager --set-enabled powertools
    # Error:
    # Problem: conflicting requests
    # - package Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.i686 is filtered out by modular filtering
    # - package Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.x86_64 is filtered out by modular filtering
    # Solution: install Judy-devel directly from downloaded rpm file:
    - yum install -y http://vault.centos.org/centos/8/PowerTools/x86_64/os/Packages/Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.x86_64.rpm
    # Use eatmydata to speed up build
    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
    - yum install -y ccache  # From EPEL
    - source /etc/profile.d/ccache.sh
    - export CCACHE_DIR="$(pwd)/.ccache"; ccache --zero-stats
    # This repository does not have any .spec files, so install dependencies based on CentOS spec file
    - yum-builddep -y mariadb-server
    - mkdir builddir; cd builddir
    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
    - make test
    # - make test-force  # mysql-test-runner takes too long, run it MTR a separate job instead
    - *rpm_listfiles
    - mkdir ../rpm; mv *.rpm ../rpm
    - ccache -s
  artifacts:
    when: always  # Must be able to see logs
    paths:
      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpm
      - builddir/_CPack_Packages/Linux/RPM/SPECS/
  cache:
    key: $MARIADB_MAJOR_VERSION
    paths:
      - .ccache

centos7:
  stage: build
  image: centos:7
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: normal
  script:
    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
    - yum-builddep -y mariadb-server
    # ..with a few extra ones, as CentOS 7 is very old and these are added in newer MariaDB releases
    - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel pcre2-devel
    - mkdir builddir; cd builddir
    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
    - make test
    # - make test-force  # mysql-test-runner takes too long, run it in a separate job instead
    - *rpm_listfiles
    - mkdir ../rpm; mv *.rpm ../rpm
  artifacts:
    when: always  # Must be able to see logs
    paths:
      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
      - rpm
      - builddir/_CPack_Packages/Linux/RPM/SPECS/

mysql-test-run:
  stage: test
  dependencies:
    - fedora
  script:
    # Install packages so tests and the dependencies install
    # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
    - yum install -y rpm/*.rpm patch diffutils
    # @TODO: Fix on packaging level for /usr/share/mariadb to work and errormsg.sys be found
    - rm -rf /usr/share/mariadb; ln -s /usr/share/mysql /usr/share/mariadb
    # mtr expects to be launched in-place and with write access to it's own directories
    - cd /usr/share/mysql-test
    # Skip failing tests
    - |
      echo "
      main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
      main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
      main.mysql_upgrade_noengine : upgrade output order does not match the expected
      " > skiplist
    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
  artifacts:
    when: always  # Also show results when tests fail
    reports:
      junit:
        - junit.xml

rpmlint:
  stage: test
  dependencies:
    - fedora
  script:
    - yum install -y rpmlint
    - rm -f rpm/*debuginfo*  # Not relevant in this test
    # Limit output to 1000 lines as Gitlab-CI max output is 4194304 bytes
    # Save everything in a log file so it can be viewed in full via artifacts
    - rpmlint --info rpm/*.rpm | tee -a rpmlint-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
  artifacts:
    when: always  # Also show results when tests fail
    paths:
      - rpmlint-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
  allow_failure: true
  # @TODO: The package is not rpmlint clean, must allow failure for now

fedora install:
  stage: test
  dependencies:
    - fedora
  script:
    - rm -f rpm/*debuginfo*  # Not relevant in this test
    # Nothing provides galera-4 on Fedora, so this step fails if built with wsrep
    - yum install -y rpm/*.rpm
    # Fedora does not support running services in Docker (like Debian packages do) so start it manually
    - /usr/bin/mariadb-install-db -u mysql
    - sudo -u mysql /usr/sbin/mariadbd & sleep 10
    # Dump database contents as is before upgrade
    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > installed-database.sql
    # Since we did a manual start, we also need to run upgrade manually
    - /usr/bin/mariadb-upgrade -u root
    # Dump database contents as is after upgrade
    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > upgraded-database.sql
    - |
      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
    - mariadb -e "SHUTDOWN;"
    - rm -rf /var/lib/mysql/*  # Clear datadir before next run
    # Start database without install-db step
    - sudo -u mysql /usr/sbin/mariadbd --skip-network --skip-grant & sleep 10
    # Dump database contents in initial state
    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > empty-database.sql
  artifacts:
    paths:
      - installed-database.sql
      - upgraded-database.sql

fedora upgrade:
  stage: test
  dependencies:
    - fedora
  script:
    - yum install -y mariadb-server
    # Fedora does not support running services in Docker (like Debian packages do) so start it manually
    - /usr/libexec/mysql-check-socket
    - /usr/libexec/mysql-prepare-db-dir
    - sudo -u mysql /usr/libexec/mysqld --basedir=/usr & sleep 10
    # Dump database contents in installed state
    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-installed-database.sql
    - /usr/libexec/mysql-check-upgrade
    # Dump database contents in upgraded state
    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-upgraded-database.sql
    - mariadb --skip-column-names -e "SELECT @@version, @@version_comment"  # Show version
    # @TODO: Upgrade from Fedora 33 MariaDB 10.4 to MariaDB.org latest does not work
    # so do this manual step to remove conflicts until packaging is fixed
    - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check mariadb-backup mariadb-connector-c-config
    - rm -f rpm/*debuginfo*  # Not relevant in this test
    - yum install -y rpm/*.rpm
    # nothing provides galera-4 on Fedora, so this step fails if built with wsrep
    - mysql -e "SHUTDOWN;"
    - /usr/bin/mariadb-install-db  # This step should not do anything on upgrades, just exit
    - sudo -u mysql /usr/sbin/mariadbd & sleep 10
    # Dump database contents in installed state
    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-installed-database.sql || true
    # The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577)
    # @TODO: Since we did a manual start, we also need to run upgrade manually
    - /usr/bin/mariadb-upgrade
    # Dump database contents in upgraded state
    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-upgraded-database.sql
    - |
      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
  artifacts:
    paths:
      - old-installed-database.sql
      - old-upgraded-database.sql
      - new-installed-database.sql
      - new-upgraded-database.sql

# Once all RPM builds and tests have passed, also run the DEB builds and tests
# @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com
# runners are too small for everything this stage does.
# build_deb:
#   stage: Salsa-CI
#   trigger:
#     include: debian/salsa-ci.yml