ci: refresh with latest lcitool manifest

This refresh switches the CI for contributors to be triggered by merge
requests. Pushing to a branch in a fork will no longer run CI pipelines,
in order to avoid consuming CI minutes. To regain the original behaviour
contributors can opt-in to a pipeline on push

   git push <remote> -o ci.variable=RUN_PIPELINE=1

This variable can also be set globally on the repository, though this is
not recommended. Upstream repo pushes to branches will run CI.

The use of containers has changed in this update, with only the upstream
repo creating containers, in order to avoid consuming contributors'
limited storage quotas. A fork with existing container images may delete
them. Containers will be rebuilt upstream when pushing commits with CI
changes to the default branch. Any other scenario with CI changes will
simply install build pre-requisite packages in a throaway environment,
using the ci/buildenv/ scripts. These scripts may also be used on a
contributor's local machines.

With pipelines triggered by merge requests, it is also now possible to
workaround the inability of contributors to run pipelines if they have
run out of CI quota. A project member can trigger a pipeline from the
merge request, which will run in context of upstream, however, note
this should only be done after reviewing the code for any malicious
CI changes.

Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>

1 files changed, 66 insertions, 211 deletions

diff --git a/ci/gitlab.yml b/ci/gitlab.yml
index 6556b0a..bdc5e0a 100644
--- a/ci/gitlab.yml
+++ b/ci/gitlab.yml
@@ -5,223 +5,78 @@
 # https://gitlab.com/libvirt/libvirt-ci
 
 
-.container_job:
-  image: docker:stable
-  stage: containers
-  needs: []
-  services:
-    - docker:dind
-  before_script:
-    - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:latest"
-    - export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt-python/ci-$NAME:latest"
-    - docker info
-    - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
-  script:
-    - docker pull "$TAG" || docker pull "$COMMON_TAG" || true
-    - docker build --cache-from "$TAG" --cache-from "$COMMON_TAG" --tag "$TAG" -f "ci/containers/$NAME.Dockerfile" ci/containers
-    - docker push "$TAG"
-  after_script:
-    - docker logout
-
-
-.gitlab_native_build_job:
-  image: $CI_REGISTRY_IMAGE/ci-$NAME:latest
-  stage: builds
-
-
-.gitlab_cross_build_job:
-  image: $CI_REGISTRY_IMAGE/ci-$NAME-cross-$CROSS:latest
-  stage: builds
-
-
-check-dco:
-  stage: sanity_checks
-  needs: []
-  image: registry.gitlab.com/libvirt/libvirt-ci/check-dco:master
-  script:
-    - /check-dco libvirt
-  except:
-    variables:
-      - $CI_PROJECT_NAMESPACE == 'libvirt'
-  variables:
-    GIT_DEPTH: 1000
-
-
-# Native container jobs
-
-x86_64-centos-stream-8-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: centos-stream-8
-
-
-x86_64-debian-10-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: debian-10
-
-
-x86_64-debian-sid-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: debian-sid
-
-
-x86_64-fedora-34-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: fedora-34
-
-
-x86_64-fedora-35-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: fedora-35
-
-
-x86_64-fedora-rawhide-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: fedora-rawhide
-
-
-x86_64-opensuse-leap-152-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: opensuse-leap-152
-
-
-x86_64-opensuse-tumbleweed-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: opensuse-tumbleweed
-
-
-x86_64-ubuntu-1804-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: ubuntu-1804
-
-
-x86_64-ubuntu-2004-container:
-  extends: .container_job
-  allow_failure: false
-  variables:
-    NAME: ubuntu-2004
-
-
-# Native build jobs
-
-x86_64-centos-stream-8:
-  extends: .native_build_job
-  needs:
-    - x86_64-centos-stream-8-container
-  allow_failure: false
-  variables:
-    NAME: centos-stream-8
-
-
-x86_64-centos-stream-8-git:
-  extends: .native_git_build_job
-  needs:
-    - x86_64-centos-stream-8-container
-  allow_failure: false
-  variables:
-    NAME: centos-stream-8
-  artifacts:
-    expire_in: 2 days
-    paths:
-      - scratch
-      - build
-
-
-x86_64-debian-10:
-  extends: .native_build_job
-  needs:
-    - x86_64-debian-10-container
-  allow_failure: false
-  variables:
-    NAME: debian-10
-
-
-x86_64-debian-sid:
-  extends: .native_build_job
-  needs:
-    - x86_64-debian-sid-container
-  allow_failure: false
-  variables:
-    NAME: debian-sid
-
-
-x86_64-fedora-34:
-  extends: .native_build_job
-  needs:
-    - x86_64-fedora-34-container
-  allow_failure: false
-  variables:
-    NAME: fedora-34
-
-
-x86_64-fedora-35:
-  extends: .native_build_job
-  needs:
-    - x86_64-fedora-35-container
-  allow_failure: false
-  variables:
-    NAME: fedora-35
-
+# Variables that can be set to control the behaviour of
+# pipelines that are run
+#
+#  - RUN_PIPELINE - force creation of a CI pipeline when
+#    pushing to a branch in a forked repository. Official
+#    CI pipelines are triggered when merge requests are
+#    created/updated. Setting this variable to a non-empty
+#    value allows CI testing prior to opening a merge request.
+#
+#  - RUN_CONTAINER_BUILDS - CI pipelines in upstream only
+#    publish containers if CI file changes are detected.
+#    Setting this variable to a non-empty value will force
+#    re-publishing, even when no file changes are detected.
+#    Typically to use from a scheduled job once a month.
+#
+#  - RUN_UPSTREAM_NAMESPACE - the upstream namespace is
+#    configured to default to 'libvirt'. When testing
+#    changes to CI it might be useful to use a different
+#    upstream. Setting this variable will override the
+#    namespace considered to be upstream.
+#
+# These can be set as git push options
+#
+#  $ git push -o ci.variable=RUN_PIPELINE=1
+#
+# Aliases can be set for common usage
+#
+#  $ git config --local alias.push-ci "push -o ci.variable=RUN_PIPELINE=1"
+#
+# Allowing the less verbose invocation
+#
+#  $ git push-ci
+#
+# Pipeline variables can also be set in the repository
+# pipeline config globally, or set against scheduled pipelines
 
-x86_64-fedora-rawhide:
-  extends: .native_build_job
-  needs:
-    - x86_64-fedora-rawhide-container
-  allow_failure: false
-  variables:
-    NAME: fedora-rawhide
 
+variables:
+  RUN_UPSTREAM_NAMESPACE: libvirt
 
-x86_64-opensuse-leap-152:
-  extends: .native_build_job
-  needs:
-    - x86_64-opensuse-leap-152-container
-  allow_failure: false
-  variables:
-    NAME: opensuse-leap-152
-    RPM: skip
 
+workflow:
+  rules:
+    # upstream+forks: Avoid duplicate pipelines on pushes, if a MR is open
+    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_OPEN_MERGE_REQUESTS'
+      when: never
 
-x86_64-opensuse-tumbleweed:
-  extends: .native_build_job
-  needs:
-    - x86_64-opensuse-tumbleweed-container
-  allow_failure: false
-  variables:
-    NAME: opensuse-tumbleweed
-    RPM: skip
+    # upstream+forks: Avoid pipelines on tag pushes
+    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG'
+      when: never
 
+    # upstream+forks: Allow pipelines in scenarios we've figured out job rules
+    - if: '$CI_PIPELINE_SOURCE =~ /^(push|merge_request_event|api|web|schedule)$/'
+      when: always
 
-x86_64-ubuntu-1804:
-  extends: .native_build_job
-  needs:
-    - x86_64-ubuntu-1804-container
-  allow_failure: false
-  variables:
-    NAME: ubuntu-1804
+    # upstream+forks: Avoid all other pipelines
+    - when: never
 
 
-x86_64-ubuntu-2004:
-  extends: .native_build_job
-  needs:
-    - x86_64-ubuntu-2004-container
-  allow_failure: false
-  variables:
-    NAME: ubuntu-2004
+debug:
+  image: docker.io/library/alpine:3
+  stage: sanity_checks
+  needs: []
+  script:
+    - printenv | sort
+  rules:
+    - if: '$RUN_DEBUG'
+      when: always
+
+include:
+  - local: '/ci/gitlab/container-templates.yml'
+  - local: '/ci/gitlab/build-templates.yml'
+  - local: '/ci/gitlab/sanity-checks.yml'
+  - local: '/ci/gitlab/containers.yml'
+  - local: '/ci/gitlab/builds.yml'