diff options
author | Thomas Haller <thaller@redhat.com> | 2023-04-13 14:30:55 +0200 |
---|---|---|
committer | Thomas Haller <thaller@redhat.com> | 2023-04-13 14:30:55 +0200 |
commit | 31b5bd72ebfc3c77ce3d80aaaaa06f1944dc7213 (patch) | |
tree | 2e791a99475c5ce729db00ad09f2bfe5183c0cdd /.gitlab-ci | |
parent | 513c7f0280ceec0728a6e72d8a412d26ec77c279 (diff) | |
parent | 229163202dfc5e51b65c7c8e02b9df1880e1fb93 (diff) | |
download | NetworkManager-31b5bd72ebfc3c77ce3d80aaaaa06f1944dc7213.tar.gz |
gitlab: merge branch 'th/gitlab-ci-tiers'
Obsoletes: https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/1595
https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/1601
Diffstat (limited to '.gitlab-ci')
-rw-r--r-- | .gitlab-ci/ci.template | 92 | ||||
-rw-r--r-- | .gitlab-ci/config.yml | 61 | ||||
-rwxr-xr-x | .gitlab-ci/run-test.sh | 89 |
3 files changed, 162 insertions, 80 deletions
diff --git a/.gitlab-ci/ci.template b/.gitlab-ci/ci.template index 02b33ff3bc..a62e9e58e4 100644 --- a/.gitlab-ci/ci.template +++ b/.gitlab-ci/ci.template @@ -17,17 +17,37 @@ .templates_sha: &template_sha ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 # see https://docs.gitlab.com/ee/ci/yaml/#includefile +{# Group distros by their common (name,) tuples.#} +{% set distro_groups = [] %} +{% for distro in distributions %} + {% set g = {'name':distro.name} %} + {% if g not in distro_groups %} + {% do distro_groups.append(g) %} + {% endif %} +{% endfor %} + +{# The "default_distro" builds our pages and is used for check-{tree,patch} tests. It is the first distro with tier 1. #} +{% set default_distro = [] %} +{% for distro in distributions %} + {% if distro.tier == 1 and default_distro|length == 0 %} + {% do default_distro.append(distro) %} + {% endif %} +{% endfor %} +{% set default_distro = default_distro[0] %} + include: -{% for distro in distributions|sort(attribute="name") %} - # {{ distro.name.capitalize() }} container builder template +{% for distro_group in distro_groups|sort(attribute='name') %} + # {{ distro_group.name.capitalize() }} container builder template - project: 'freedesktop/ci-templates' ref: *template_sha - file: '/templates/{{distro.name}}.yml' + file: '/templates/{{distro_group.name}}.yml' {% endfor %} stages: - prep - - test + - tier1 + - tier2 + - tier3 - deploy - triage - container_clean @@ -41,17 +61,17 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". -{% for distro in distributions %} - {{"%-13s"| format(distro.name.upper() + '_TAG:')}}'{{distro.tag}}-{{ +{% for distro_group in distro_groups|sort(attribute='name') %} + {{"%-13s"| format(distro_group.name.upper() + '_TAG:')}}'tag-{{ (ci_fairy.hashfiles('./.gitlab-ci/config.yml', './.gitlab-ci/ci.template', - './.gitlab-ci/' + distro.base_type + '-install.sh', - './contrib/' + distro.base_type + '/REQUIRED_PACKAGES'))[0:12] + './.gitlab-ci/' + base_types[distro_group.name] + '-install.sh', + './contrib/' + base_types[distro_group.name] + '/REQUIRED_PACKAGES'))[0:12] }}' {% endfor %} -{% for distro in distributions %} - {{"%-13s"| format(distro.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{distro.base_type}}-install.sh' +{% for distro_group in distro_groups|sort(attribute='name') %} + {{"%-13s"| format(distro_group.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{base_types[distro_group.name]}}-install.sh' {% endfor %} .nm_artifacts: @@ -84,7 +104,7 @@ variables: {% for distro in distributions %} {% for version in distro.versions %} -{{distro.name}}:{{version}}@container-prep: +tier{{distro.tier}}:{{distro.name}}:{{version}}@prep: extends: - .fdo.container-build@{{distro.name}} stage: prep @@ -93,6 +113,9 @@ variables: FDO_DISTRIBUTION_VERSION: '{{version}}' FDO_DISTRIBUTION_TAG: ${{distro.name.upper()}}_TAG FDO_DISTRIBUTION_EXEC: ${{distro.name.upper()}}_EXEC +{% if distro.tier > 1 %} + when: manual +{% endif %} {% endfor %} {% endfor %} @@ -151,7 +174,6 @@ variables: ################################################################# .build@template: - stage: test script: - env - r=0 @@ -174,12 +196,25 @@ t_{{distro.name}}:{{version}}: extends: - .build@template - .fdo.distribution-image@{{distro.name}} -{% if distro.name == pages_build.name and - version == pages_build.version %} +{% if distro == default_distro %} - .nm_artifacts {% else %} - .nm_artifacts_debug {% endif %} + stage: tier{{distro.tier}} +{% if distro.tier <= 1 %} + parallel: + matrix: + - NM_TEST_SELECT_RUN: + - autotools+gcc+docs+valgrind + - meson+gcc+docs+valgrind + - autotools+clang + - meson+clang + - rpm+autotools + - rpm+meson + - tarball + - subtree +{% endif %} variables: FDO_DISTRIBUTION_VERSION: '{{version}}' FDO_DISTRIBUTION_TAG: ${{distro.name.upper()}}_TAG @@ -190,10 +225,7 @@ t_{{distro.name}}:{{version}}: {% endfor %} {% endif %} needs: - - "{{distro.name}}:{{version}}@container-prep" -{% if not version in distro.get('always', []) and (distro.name != pages_build.name or version != pages_build.version) %} - when: manual -{% endif %} + - "tier{{distro.tier}}:{{distro.name}}:{{version}}@prep" {% endfor %} {% endfor %} @@ -205,26 +237,26 @@ t_{{distro.name}}:{{version}}: check-patch: extends: - - .fdo.distribution-image@{{pages_build.name}} + - .fdo.distribution-image@{{default_distro.name}} variables: - FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}' - FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG + FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' + FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{pages_build.name}}:{{pages_build.version}}@container-prep" - stage: test + - "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep" + stage: tier1 script: - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh allow_failure: true check-tree: extends: - - .fdo.distribution-image@{{pages_build.name}} + - .fdo.distribution-image@{{default_distro.name}} variables: - FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}' - FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG + FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' + FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{pages_build.name}}:{{pages_build.version}}@container-prep" - stage: test + - "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep" + stage: tier1 script: - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check - date '+%Y%m%d-%H%M%S'; git ls-files -z -- 'po/*.po' | xargs -0 -n1 msgfmt -vc @@ -242,9 +274,9 @@ pages: only: - main dependencies: - - t_{{pages_build.name}}:{{pages_build.version}} + - "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]" needs: - - t_{{pages_build.name}}:{{pages_build.version}} + - "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]" triage:issues: stage: triage diff --git a/.gitlab-ci/config.yml b/.gitlab-ci/config.yml index 117785c8c3..8aafe0c48b 100644 --- a/.gitlab-ci/config.yml +++ b/.gitlab-ci/config.yml @@ -7,16 +7,29 @@ # https://gitlab.freedesktop.org/freedesktop/ci-templates # -# We're happy to rebuild all containers when one changes. -.default_tag: &default_tag '2023-01-18.0' - +# Some distros are fairly similar, and we reuse similar scripts. +# The base type maps the distro name to their base. +base_types: + fedora: fedora + centos: fedora + debian: debian + ubuntu: debian + alpine: alpine # The list of all distributions we want to create job for. -# The template generates manual jobs for all these. distributions: - name: fedora - tag: *default_tag - base_type: fedora + # The first tier:1 in the list is used to build the pages and check-{tree,patch} + tier: 1 + versions: + - '37' + - name: fedora + tier: 2 + versions: + - '36' + - '38' + - name: fedora + tier: 3 versions: - '30' - '31' @@ -24,12 +37,8 @@ distributions: - '33' - '34' - '35' - - '36' - - '37' - - '38' - name: ubuntu - tag: *default_tag - base_type: debian + tier: 2 versions: - '18.04' - '20.04' @@ -37,33 +46,31 @@ distributions: - 'devel' - 'rolling' - name: debian - tag: *default_tag - base_type: debian + tier: 2 versions: - - '9' - '10' - '11' - - 'testing' - 'sid' + - name: debian + tier: 3 + versions: + - '9' + - 'testing' - name: centos - tag: *default_tag - base_type: fedora + tier: 2 versions: - '7.5.1804' + - '7.9.2009' + - '8.1.1911' + - '8.3.2011' + - name: centos + tier: 3 + versions: - '7.6.1810' - '7.7.1908' - '7.8.2003' - - '7.9.2009' - - '8.1.1911' - '8.2.2004' - - '8.3.2011' - name: alpine - tag: *default_tag - base_type: alpine + tier: 2 versions: - 'latest' - -# specifies which of the above distros is used as source for pages -pages_build: - name: fedora - version: '37' diff --git a/.gitlab-ci/run-test.sh b/.gitlab-ci/run-test.sh index d991010dc1..50d0903619 100755 --- a/.gitlab-ci/run-test.sh +++ b/.gitlab-ci/run-test.sh @@ -2,6 +2,11 @@ set -ex +die() { + printf "%s\n" "$*" >&2 + exit 1 +} + export PAGER=cat export OMP_NUM_THREADS=1 @@ -44,18 +49,58 @@ meson --version # to run that test as part of the build. Disable it. export NMTST_SKIP_CHECK_GITLAB_CI=1 +# Assert that "$1" is one of the valid values for NM_TEST_SELECT_RUN. die() otherwise. +check_run_assert() { + { set +x; } 2>/dev/null + local run="$1" + local a + + # These are the supported $NM_TEST_SELECT_RUN values. + local _CHECK_RUN_LIST=( + autotools+gcc+docs+valgrind + meson+gcc+docs+valgrind + autotools+clang + meson+clang + autotools+gcc+docs+el7+py2 + rpm+autotools + rpm+meson + tarball + subtree + + all + none + ) + + if [ "$run" = all ] ; then + set -x + return 0 + fi + + for a in "${_CHECK_RUN_LIST[@]}" ; do + if [ "$a" = "$run" ] ; then + set -x + return 0 + fi + done + die "invalid NM_TEST_SELECT_RUN value \"$1\"" +} + +[ -z "$NM_TEST_SELECT_RUN" ] && NM_TEST_SELECT_RUN=all +check_run_assert "$NM_TEST_SELECT_RUN" + check_run() { local test_no="$1" + check_run_assert "$test_no" + # Usually, we run the build several times. However, for testing # the build script manually, it can be useful to explicitly select # one step to run. For example, if step 3 is known to fail, you - # can still manually run step 4 by setting NM_TEST_SELECT_RUN=4. + # can still manually run step A by setting NM_TEST_SELECT_RUN=A. - test -z "$NM_TEST_SELECT_RUN" -o "$NM_TEST_SELECT_RUN" = "$test_no" + test "$NM_TEST_SELECT_RUN" = all -o "$NM_TEST_SELECT_RUN" = "$test_no" } - check_run_clean() { if ! check_run "$1" ; then return 1 @@ -64,21 +109,21 @@ check_run_clean() { return 0 } -if check_run_clean 1 ; then +if check_run_clean autotools+gcc+docs+valgrind ; then BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh mv build/INST/share/gtk-doc/html "$ARTIFACT_DIR/docs-html" fi -check_run_clean 2 && BUILD_TYPE=meson CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh -check_run_clean 3 && BUILD_TYPE=autotools CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh -check_run_clean 4 && BUILD_TYPE=meson CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh +check_run_clean meson+gcc+docs+valgrind && BUILD_TYPE=meson CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh +check_run_clean autotools+clang && BUILD_TYPE=autotools CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh +check_run_clean meson+clang && BUILD_TYPE=meson CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh -check_run_clean 5 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh +check_run_clean autotools+gcc+docs+el7+py2 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh -check_run_clean 6 && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson -check_run_clean 7 && test $IS_FEDORA = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson +check_run_clean rpm+autotools && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson +check_run_clean rpm+meson && test $IS_FEDORA = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson -if check_run_clean 8 && [ "$NM_BUILD_TARBALL" = 1 ]; then +if check_run_clean tarball && [ "$NM_BUILD_TARBALL" = 1 ]; then SIGN_SOURCE=0 ./contrib/fedora/rpm/build_clean.sh -r mv ./NetworkManager-1*.tar.xz "$ARTIFACT_DIR/" mv ./contrib/fedora/rpm/latest/SRPMS/NetworkManager-1*.src.rpm "$ARTIFACT_DIR/" @@ -110,7 +155,7 @@ test_subtree() { popd } -if check_run_clean 10; then +if check_run_clean subtree; then for d in c-list c-rbtree c-siphash c-stdaux n-acd n-dhcp4 ; do for cc in gcc clang; do test_subtree "$d" "$cc" @@ -120,18 +165,16 @@ fi ############################################################################### -if [ -z "$NM_TEST_SELECT_RUN" ] ; then +if [ "$NM_BUILD_TARBALL" = 1 ]; then do_clean - if [ "$NM_BUILD_TARBALL" = 1 ]; then - if check_run 1 ; then - mv "$ARTIFACT_DIR/docs-html/" ./ - fi - if check_run 8 ; then - mv \ - "$ARTIFACT_DIR"/NetworkManager-1*.tar.xz \ - "$ARTIFACT_DIR"/NetworkManager-1*.src.rpm \ - ./ - fi + if check_run autotools+gcc+docs+valgrind ; then + mv "$ARTIFACT_DIR/docs-html/" ./ + fi + if check_run tarball ; then + mv \ + "$ARTIFACT_DIR"/NetworkManager-1*.tar.xz \ + "$ARTIFACT_DIR"/NetworkManager-1*.src.rpm \ + ./ fi fi |