diff options
Diffstat (limited to '.gitlab/ci')
| -rw-r--r-- | .gitlab/ci/cache-repo.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/cng.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/dev-fixtures.gitlab-ci.yml | 14 | ||||
| -rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 3 | ||||
| -rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 89 | ||||
| -rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 74 | ||||
| -rw-r--r-- | .gitlab/ci/memory.gitlab-ci.yml | 6 | ||||
| -rw-r--r-- | .gitlab/ci/pages.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 7 | ||||
| -rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 223 | ||||
| -rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 29 | ||||
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 85 | ||||
| -rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 204 | ||||
| -rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 4 | ||||
| -rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 13 |
15 files changed, 340 insertions, 414 deletions
diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml index ecbed0ed6c8..a091785dec3 100644 --- a/.gitlab/ci/cache-repo.gitlab-ci.yml +++ b/.gitlab/ci/cache-repo.gitlab-ci.yml @@ -21,7 +21,6 @@ cache-repo: extends: .cache-repo:rules image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine stage: sync - allow_failure: true variables: GIT_STRATEGY: none TAR_FILENAME: /tmp/gitlab-master.tar diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index 2450e346569..d7699de74e2 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -3,7 +3,6 @@ cloud-native-image: image: ruby:2.6-alpine dependencies: [] stage: post-test - allow_failure: true variables: GIT_DEPTH: "1" script: diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml index 27ceb6f37db..fc3678a7d17 100644 --- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml +++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml @@ -1,11 +1,11 @@ .run-dev-fixtures: extends: - .default-retry - - .default-cache + - .rails-cache - .default-before_script - .use-pg11 stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] variables: FIXTURE_PATH: "db/fixtures/development" SEED_CYCLE_ANALYTICS: "true" @@ -19,8 +19,9 @@ run-dev-fixtures: - .run-dev-fixtures - .dev-fixtures:rules:ee-and-foss script: - - scripts/gitaly-test-spawn - - RAILS_ENV=test bundle exec rake db:seed_fu + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" run-dev-fixtures-ee: extends: @@ -28,6 +29,7 @@ run-dev-fixtures-ee: - .dev-fixtures:rules:ee-only - .use-pg11-ee script: - - scripts/gitaly-test-spawn + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - cp ee/db/fixtures/development/* $FIXTURE_PATH - - RAILS_ENV=test bundle exec rake db:seed_fu + - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index b8a66decbb7..50dbef44598 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -2,7 +2,6 @@ extends: - .default-retry - .docs:rules:review-docs - allow_failure: true image: ruby:2.6-alpine stage: review dependencies: [] @@ -68,7 +67,7 @@ graphql-reference-verify: - .docs:rules:graphql-reference-verify - .use-pg11 stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] script: - bundle exec rake gitlab:graphql:check_docs - bundle exec rake gitlab:graphql:check_schema diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 2b22162b0c2..6e9119f295a 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -15,8 +15,7 @@ - .default-retry - .default-before_script - .assets-compile-cache - - .use-docker-in-docker - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1 stage: prepare variables: NODE_ENV: "production" @@ -34,20 +33,16 @@ paths: - webpack-report/ - assets-compile.log - # We consume these files in GitLab UI for integration tests: - # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1 - - public/assets/application-*.css - - public/assets/application-*.css.gz + # These assets are used in multiple locations: + # - in `build-assets-image` job to create assets image for packaging systems + # - GitLab UI for integration tests: https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1 + - public/assets when: always script: - node --version - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline - free -m - time bin/rake gitlab:assets:compile > assets-compile.log 2>&1 - # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists - # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines - # https://gitlab.com/gitlab-org/gitlab/issues/208389 - - time scripts/build_assets_image - scripts/clean-old-cached-assets - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here @@ -65,6 +60,20 @@ gitlab:assets:compile pull-cache: cache: policy: pull +build-assets-image: + extends: + - .use-kaniko + - .frontend:rules:gitlab-assets-compile-pull-cache + stage: build-images + needs: ["gitlab:assets:compile pull-cache"] + variables: + GIT_DEPTH: "1" + script: + # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists + # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines + # https://gitlab.com/gitlab-org/gitlab/issues/208389 + - scripts/build_assets_image + .compile-assets-metadata: extends: - .default-retry @@ -127,16 +136,15 @@ compile-assets pull-cache as-if-foss: .frontend-fixtures-base: extends: - .default-retry - - .default-cache + - .rails-cache - .default-before_script - .use-pg11 stage: fixtures - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: ["setup-test-env", "compile-assets pull-cache"] script: - - date - - scripts/gitaly-test-spawn - - date - - bundle exec rake frontend:fixtures + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - run_timed_command "bundle exec rake frontend:fixtures" artifacts: name: frontend-fixtures expire_in: 31d @@ -154,7 +162,7 @@ frontend-fixtures: frontend-fixtures-as-if-foss: extends: - .frontend-fixtures-base - - .frontend:rules:default-frontend-jobs-no-foss + - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss .frontend-job-base: @@ -197,7 +205,7 @@ karma: karma-as-if-foss: extends: - .karma-base - - .frontend:rules:default-frontend-jobs-no-foss + - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss needs: ["frontend-fixtures-as-if-foss"] @@ -229,10 +237,24 @@ jest: junit: junit_jest.xml parallel: 2 +jest-integration: + extends: + - .frontend-job-base + - .frontend:rules:default-frontend-jobs + script: + - date + - yarn jest:integration --ci + needs: ["frontend-fixtures"] + cache: + key: jest-integration + paths: + - tmp/cache/jest/ + policy: pull-push + jest-as-if-foss: extends: - .jest-base - - .frontend:rules:default-frontend-jobs-no-foss + - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss needs: ["frontend-fixtures-as-if-foss"] cache: @@ -241,11 +263,13 @@ jest-as-if-foss: coverage-frontend: extends: - .default-retry - - .frontend:rules:default-frontend-jobs-no-foss + - .yarn-cache + - .frontend:rules:ee-mr-and-master-only needs: ["jest"] stage: post-test before_script: - - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline + - source scripts/utils.sh + - retry yarn install --frozen-lockfile script: - yarn node scripts/frontend/merge_coverage_frontend.js artifacts: @@ -254,8 +278,7 @@ coverage-frontend: paths: - coverage-frontend/ cache: - paths: - - .yarn-cache/ + policy: pull .qa-frontend-node: extends: @@ -279,22 +302,32 @@ qa-frontend-node:10: image: node:dubnium qa-frontend-node:latest: - extends: .qa-frontend-node + extends: + - .qa-frontend-node + - .frontend:rules:qa-frontend-node-latest image: node:latest - allow_failure: true webpack-dev-server: extends: - .default-retry - - .default-cache - .frontend:rules:default-frontend-jobs stage: test - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: [] variables: WEBPACK_MEMORY_TEST: "true" WEBPACK_VENDOR_DLL: "true" + cache: + key: + files: + - yarn.lock + prefix: "v1" + paths: + - node_modules/ + - tmp/cache/webpack-dlls/ script: - - yarn webpack-vendor + - source scripts/utils.sh + - retry yarn install --frozen-lockfile + - retry yarn webpack-vendor - node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js artifacts: name: webpack-dev-server diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 66b7c47efcf..e6619ff2b6d 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -21,7 +21,7 @@ # Jobs that only need to pull cache .default-cache: cache: - key: "debian-stretch-ruby-2.6.5-pg11-node-12.x" + key: "debian-stretch-ruby-2.6.6-pg11-node-12.x" paths: - .go/pkg/mod - vendor/ruby @@ -29,65 +29,39 @@ - vendor/gitaly-ruby policy: pull -.use-pg9: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" - services: - - name: postgres:9.6.17 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - variables: - POSTGRES_HOST_AUTH_METHOD: trust +.rails-cache: cache: - key: "debian-stretch-ruby-2.6.5-pg9-node-12.x" + key: + files: + - Gemfile.lock + - GITALY_SERVER_VERSION + prefix: "ruby-go-cache-v1" + paths: + - vendor/ruby + - vendor/gitaly-ruby + - .go/pkg/mod + policy: pull -.use-pg10: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" - services: - - name: postgres:10.12 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - variables: - POSTGRES_HOST_AUTH_METHOD: trust +.yarn-cache: cache: - key: "debian-stretch-ruby-2.6.5-pg10-node-12.x" + key: + files: + - yarn.lock + prefix: "v1" + paths: + - node_modules/ .use-pg11: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine variables: POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg11-node-12.x" - -.use-pg9-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" - services: - - name: postgres:9.6.17 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - - name: elasticsearch:6.4.2 - variables: - POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg9-node-12.x" - -.use-pg10-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" - services: - - name: postgres:10.12 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - - name: elasticsearch:6.4.2 - variables: - POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg10-node-12.x" .use-pg11-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -95,15 +69,13 @@ - name: elasticsearch:6.4.2 variables: POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg11-node-12.x" -# Pin kaniko to v0.16.0 due to https://github.com/GoogleContainerTools/kaniko/issues/1162 .use-kaniko: image: - name: gcr.io/kaniko-project/executor:debug-v0.16.0 + name: gcr.io/kaniko-project/executor:debug-v0.20.0 entrypoint: [""] before_script: + - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json .as-if-foss: diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index af75ff257ea..79dfc88d132 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -8,7 +8,7 @@ memory-static: extends: .only-code-memory-job-base stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] variables: SETUP_DB: "false" script: @@ -28,6 +28,7 @@ memory-static: - tmp/memory_*.txt reports: metrics: tmp/memory_metrics.txt + expire_in: 31d # Show memory usage caused by invoking require per gem. # Unlike `memory-static`, it hits the app with one request to ensure that any last minute require-s have been called. @@ -38,7 +39,7 @@ memory-on-boot: - .only-code-memory-job-base - .use-pg11 stage: test - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: ["setup-test-env", "compile-assets pull-cache"] variables: NODE_ENV: "production" RAILS_ENV: "production" @@ -54,3 +55,4 @@ memory-on-boot: - tmp/memory_*.txt reports: metrics: tmp/memory_on_boot_metrics.txt + expire_in: 31d diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index 38d79ddb090..218ec7043d9 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -15,3 +15,4 @@ pages: artifacts: paths: - public + expire_in: 31d diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 8a8f66a4643..40ef13dd92b 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -23,7 +23,7 @@ qa:internal: qa:internal-as-if-foss: extends: - .qa-job-base - - .qa:rules:ee-only + - .qa:rules:as-if-foss - .as-if-foss script: - bundle exec rspec @@ -38,7 +38,7 @@ qa:selectors: qa:selectors-as-if-foss: extends: - qa:selectors - - .qa:rules:ee-only + - .qa:rules:as-if-foss - .as-if-foss .package-and-qa-base: @@ -58,6 +58,5 @@ package-and-qa: needs: - job: build-qa-image artifacts: false - - job: gitlab:assets:compile pull-cache + - job: build-assets-image artifacts: false - allow_failure: true diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 28ec96b838a..e8087aebcef 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,48 +1,43 @@ .rails:needs:setup-and-assets: - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: ["setup-test-env", "compile-assets pull-cache"] .rails-job-base: extends: - .default-retry - - .default-cache - .default-before_script + - .rails-cache -#################### -# EE and FOSS jobs # -.base-setup-test-env: +####################################################### +# EE/FOSS: default refs (MRs, master, schedules) jobs # +setup-test-env: extends: - .rails-job-base + - .rails:rules:default-refs-code-backstage-qa + - .use-pg11 stage: prepare + variables: + GITLAB_TEST_EAGER_LOAD: "0" script: - - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' - - scripts/gitaly-test-build # Do not use 'bundle exec' here + - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'" + - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here + - rm tmp/tests/gitaly/.ruby-bundle # This file prevents gems from being installed even if vendor/gitaly-ruby is missing artifacts: expire_in: 7d paths: - - tmp/tests - config/secrets.yml - - vendor/gitaly-ruby + - tmp/tests/gitaly + - tmp/tests/gitlab-elasticsearch-indexer + - tmp/tests/gitlab-shell + - tmp/tests/gitlab-test-fork + - tmp/tests/gitlab-test-fork_bare + - tmp/tests/gitlab-test + - tmp/tests/gitlab-workhorse + - tmp/tests/repositories + - tmp/tests/second_storage + when: always cache: policy: pull-push -setup-test-env pg11: - extends: - - .base-setup-test-env - - .rails:rules:default-refs-code-backstage-qa - - .use-pg11 - -setup-test-env pg10: - extends: - - .base-setup-test-env - - .rails:rules:master-refs-code-backstage - - .use-pg10 - -setup-test-env pg9: - extends: - - .base-setup-test-env - - .rails:rules:nightly-master-refs-code-backstage - - .use-pg9 - static-analysis: extends: - .rails-job-base @@ -55,7 +50,7 @@ static-analysis: script: - scripts/static-analysis cache: - key: "ruby-2.6.5-pg11-rubocop" + key: "ruby-2.6.6-pg11-rubocop" paths: - vendor/ruby - tmp/rubocop_cache @@ -64,8 +59,8 @@ static-analysis: downtime_check: extends: - .rails-job-base - - .rails:needs:setup-and-assets - .rails:rules:downtime_check + needs: ["setup-test-env"] stage: test variables: SETUP_DB: "false" @@ -75,8 +70,10 @@ downtime_check: .rspec-base: extends: .rails-job-base stage: test - needs: ["setup-test-env pg11", "retrieve-tests-metadata", "compile-assets pull-cache"] + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"] script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" artifacts: @@ -89,7 +86,7 @@ downtime_check: - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ - - junit_rspec.xml + - log/*.log reports: junit: junit_rspec.xml @@ -101,6 +98,8 @@ downtime_check: .rspec-base-migration: script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration" @@ -133,7 +132,7 @@ rspec fast_spec_helper: - .rails:rules:ee-and-foss - .use-pg11 stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] db:migrate:reset: extends: .db-job-base @@ -141,28 +140,28 @@ db:migrate:reset: - bundle exec rake db:migrate:reset db:check-schema: - extends: .db-job-base + extends: + - .db-job-base + - .rails:rules:ee-mr-and-master-only script: - source scripts/schema_changed.sh -db:migrate-from-v11.11.0: +db:migrate-from-v12.10.0: extends: .db-job-base variables: SETUP_DB: "false" script: - - export PROJECT_TO_CHECKOUT="gitlab-foss" - - export TAG_TO_CHECKOUT="v11.11.0" - - '[[ ! -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"' - - '[[ ! -d "ee/" ]] || export TAG_TO_CHECKOUT="v11.11.0-ee"' + - export PROJECT_TO_CHECKOUT="gitlab" + - export TAG_TO_CHECKOUT="v12.10.0-ee" + - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab-foss"' + - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="v12.10.0"' - git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT - git checkout -f FETCH_HEAD - - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile - - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile - bundle update google-protobuf grpc bootsnap - bundle install $BUNDLE_INSTALL_FLAGS - date - cp config/gitlab.yml.example config/gitlab.yml - - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - bundle exec rake db:drop db:create db:structure:load db:seed_fu - date - git checkout -f $CI_COMMIT_SHA - bundle install $BUNDLE_INSTALL_FLAGS @@ -186,23 +185,24 @@ gitlab:setup: # db/fixtures/development/04_project.rb thanks to SIZE=1 below - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git - - scripts/gitaly-test-spawn + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup artifacts: when: on_failure expire_in: 1d paths: - - log/development.log + - log/*.log rspec:coverage: extends: - .rails-job-base - - .rails:rules:ee-only + - .rails:rules:ee-mr-and-master-only stage: post-test # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) # so we use `dependencies` here. dependencies: - - setup-test-env pg11 + - setup-test-env - rspec migration pg11 - rspec unit pg11 - rspec integration pg11 @@ -231,104 +231,11 @@ rspec:coverage: - coverage/index.html - coverage/assets/ - tmp/memory_test/ -# EE and FOSS jobs # -#################### - -#################### -# master-only jobs # -.rspec-base-pg10: - extends: - - .rspec-base - - .rails:rules:master-refs-code-backstage - - .use-pg10 - needs: ["setup-test-env pg10", "retrieve-tests-metadata", "compile-assets pull-cache"] +# EE/FOSS: default refs (MRs, master, schedules) jobs # +####################################################### -rspec migration pg10: - extends: - - .rspec-base-pg10 - - .rspec-base-migration - parallel: 5 - -rspec unit pg10: - extends: .rspec-base-pg10 - parallel: 20 - -rspec integration pg10: - extends: .rspec-base-pg10 - parallel: 8 - -rspec system pg10: - extends: .rspec-base-pg10 - parallel: 24 -# master-only jobs # -#################### - -###################### -# nightly-only jobs # -.rspec-base-pg9: - extends: - - .rspec-base - - .rails:rules:nightly-master-refs-code-backstage - - .use-pg9 - needs: ["setup-test-env pg9", "retrieve-tests-metadata", "compile-assets pull-cache"] - -rspec migration pg9: - extends: - - .rspec-base-pg9 - - .rspec-base-migration - parallel: 5 - -rspec unit pg9: - extends: .rspec-base-pg9 - parallel: 20 - -rspec integration pg9: - extends: .rspec-base-pg9 - parallel: 8 - -rspec system pg9: - extends: .rspec-base-pg9 - parallel: 24 -# nightly-only jobs # -##################### - -####################### -# EE master-only jobs # -.rspec-ee-base-pg10: - extends: - - .rspec-base-ee - - .use-pg10-ee - needs: ["setup-test-env pg10", "retrieve-tests-metadata", "compile-assets pull-cache"] - -rspec-ee migration pg10: - extends: - - .rspec-ee-base-pg10 - - .rspec-base-migration - - .rails:rules:master-refs-code-backstage - parallel: 2 - -rspec-ee unit pg10: - extends: - - .rspec-ee-base-pg10 - - .rails:rules:master-refs-code-backstage - parallel: 10 - -rspec-ee integration pg10: - extends: - - .rspec-ee-base-pg10 - - .rails:rules:master-refs-code-backstage - parallel: 4 - -rspec-ee system pg10: - extends: - - .rspec-ee-base-pg10 - - .rails:rules:master-refs-code-backstage - parallel: 6 -# EE master-only jobs # -####################### - -################ -# EE-only jobs # +################################################## +# EE: default refs (MRs, master, schedules) jobs # .rspec-base-ee: extends: - .rspec-base @@ -336,10 +243,11 @@ rspec-ee system pg10: .rspec-base-pg11-as-if-foss: extends: - - .rspec-base-ee + - .rspec-base + - .rails:rules:as-if-foss - .as-if-foss - .use-pg11 - needs: ["setup-test-env pg11", "retrieve-tests-metadata", "compile-assets pull-cache as-if-foss"] + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache as-if-foss"] .rspec-ee-base-pg11: extends: @@ -385,6 +293,8 @@ rspec-ee system pg11: .rspec-ee-base-geo: extends: .rspec-base-ee script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - scripts/prepare_postgres_fdw.sh - rspec_paralellized_job "--tag ~quarantine --tag geo" @@ -411,26 +321,5 @@ db:rollback geo: script: - bundle exec rake geo:db:migrate VERSION=20170627195211 - bundle exec rake geo:db:migrate -# EE-only jobs # -################ - -######################## -# EE nightly-only jobs # -.rspec-ee-base-geo-pg9: - extends: - - .rspec-ee-base-geo - - .use-pg9-ee - - .rails:rules:nightly-master-refs-code-backstage-ee-only - needs: ["setup-test-env pg9", "retrieve-tests-metadata", "compile-assets pull-cache"] - -rspec-ee unit pg9 geo: - extends: .rspec-ee-base-geo-pg9 - parallel: 2 - -rspec-ee integration pg9 geo: - extends: .rspec-ee-base-geo-pg9 - -rspec-ee system pg9 geo: - extends: .rspec-ee-base-geo-pg9 -# EE nightly-only jobs # -######################## +# EE: default refs (MRs, master, schedules) jobs # +################################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 61915aa798e..153334e1aff 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -14,11 +14,7 @@ code_quality: - .use-docker-in-docker stage: test needs: [] - allow_failure: true variables: - # emptying DOCKER_HOST so it can be detected properly on kubernetes executor - # with the script below - DOCKER_HOST: "" CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.9" script: - | @@ -47,12 +43,10 @@ code_quality: extends: - .default-retry - .reports:rules:sast - - .use-docker-in-docker stage: test # `needs: []` starts the job immediately in the pipeline # https://docs.gitlab.com/ee/ci/yaml/README.html#needs needs: [] - allow_failure: true artifacts: paths: - gl-sast-report.json # GitLab-specific @@ -60,10 +54,6 @@ code_quality: sast: gl-sast-report.json expire_in: 1 week # GitLab-specific variables: - # emptying DOCKER_HOST so it can be detected properly on kubernetes executor - # with the script below - DOCKER_HOST: "" - DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SAST_ANALYZER_IMAGE_TAG: 2 @@ -82,15 +72,11 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" -kubesec-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" - -nodejs-scan-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" +# Temporary disabled as it's constantly failing. See https://gitlab.com/gitlab-org/gitlab/-/issues/213769. +# nodejs-scan-sast: +# extends: .sast +# image: +# name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" secrets-sast: extends: .sast @@ -108,11 +94,7 @@ dependency_scanning: stage: test needs: [] variables: - # emptying DOCKER_HOST so it can be detected properly on kubernetes executor - # with the script below - DOCKER_HOST: "" DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec" # GitLab-specific - allow_failure: true script: - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | @@ -183,7 +165,6 @@ dast: # DAST_USERNAME_FIELD: "user[login]" # DAST_PASSWORD_FIELD: "user[passowrd]" DAST_VERSION: 1 - allow_failure: true script: - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' # To be done in a later iteration diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 06710b3b9dd..46a281cd48f 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,70 +1,36 @@ -.review-docker: - extends: - - .default-retry - - .use-docker-in-docker - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6 - variables: - GITLAB_EDITION: "ce" - build-qa-image: extends: - .use-kaniko - .default-retry - - .review:rules:mr-and-schedule-auto - stage: prepare + - .review:rules:build-qa-image + stage: build-images + needs: [] script: - - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true -.review-cleanup-base: +review-cleanup: extends: - .default-retry - .review:rules:review-cleanup + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 stage: prepare - allow_failure: true environment: name: review/auto-cleanup action: stop before_script: - source scripts/utils.sh + - source scripts/review_apps/gcp_cleanup.sh - install_gitlab_gem + - setup_gcp_dependencies script: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb - -review-cleanup: - extends: - - .review-cleanup-base - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - -review-cleanup-helm3: - extends: - - .review-cleanup-base - variables: - HELM_3: 1 - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 - -review-gcp-cleanup: - extends: - - .review:rules:review-gcp-cleanup - stage: prepare - image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest - allow_failure: true - environment: - name: review/auto-gcp-cleanup - action: stop - before_script: - - gcloud auth activate-service-account --key-file=$REVIEW_APPS_GCP_CREDENTIALS - - gcloud config set project $REVIEW_APPS_GCP_PROJECT - - apt-get install -y jq - - source scripts/review_apps/gcp_cleanup.sh - script: - gcp_cleanup review-build-cng: extends: - .default-retry - - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + - .review:rules:review-build-cng image: ruby:2.6-alpine stage: review-prepare before_script: @@ -87,8 +53,7 @@ review-build-cng: variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v3.2.2" - GITLAB_EDITION: "ce" + GITLAB_HELM_CHART_REF: "v3.3.3" environment: name: review/${CI_COMMIT_REF_NAME} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} @@ -102,9 +67,7 @@ review-deploy: stage: review dependencies: [] resource_group: "review/${CI_COMMIT_REF_NAME}" - allow_failure: true before_script: - - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) @@ -146,28 +109,28 @@ review-deploy: review-stop-failed-deployment: extends: - .review-stop-base - - .review:rules:mr-only-auto + - .review:rules:review-stop-failed-deployment stage: prepare script: - delete_failed_release - - delete_helm2_release review-stop: extends: - .review-stop-base - .review:rules:mr-only-manual stage: review - allow_failure: true script: - delete_release .review-qa-base: - extends: .review-docker + extends: + - .default-retry + - .use-docker-in-docker + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6 stage: qa # This is needed so that manual jobs with needs don't block the pipeline. # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. dependencies: ["review-deploy"] - allow_failure: true variables: QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" QA_CAN_TEST_GIT_PROTOCOL_V2: "false" @@ -179,8 +142,7 @@ review-stop: GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" before_script: - - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - echo "${QA_IMAGE}" @@ -196,7 +158,7 @@ review-stop: review-qa-smoke: extends: - .review-qa-base - - .review:rules:mr-only-auto-if-frontend-manual-otherwise + - .review:rules:review-qa-smoke script: - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" @@ -221,7 +183,6 @@ review-performance: # This is needed so that manual jobs with needs don't block the pipeline. # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. dependencies: ["review-deploy"] - allow_failure: true before_script: - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" @@ -237,6 +198,7 @@ review-performance: - sitespeed-results/ reports: performance: performance.json + expire_in: 31d parallel-spec-reports: extends: @@ -244,7 +206,6 @@ parallel-spec-reports: image: ruby:2.6-alpine stage: post-qa dependencies: ["review-qa-all"] - allow_failure: true variables: NEW_PARALLEL_SPECS_REPORT: qa/report-new.html BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" @@ -263,17 +224,19 @@ parallel-spec-reports: - qa/gitlab-qa-run-* reports: junit: qa/gitlab-qa-run-*/**/rspec-*.xml + expire_in: 31d danger-review: extends: - .default-retry - - .default-cache + - .yarn-cache - .review:rules:danger image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test needs: [] script: - - git version - - node --version - - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline + - source scripts/utils.sh + - retry yarn install --frozen-lockfile - danger --fail-on-errors=true --verbose + cache: + policy: pull diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 79ae7823853..383aca0043b 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -16,6 +16,15 @@ .if-master-refs: &if-master-refs if: '$CI_COMMIT_REF_NAME == "master"' +.if-master-push: &if-master-push + if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "push"' + +.if-master-schedule-2-hourly: &if-master-schedule-2-hourly + if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' + +.if-master-schedule-nightly: &if-master-schedule-nightly + if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "nightly"' + .if-auto-deploy-branches: &if-auto-deploy-branches if: '$CI_COMMIT_BRANCH =~ /^\d+-\d+-auto-deploy-\d+$/' @@ -25,8 +34,11 @@ .if-merge-request: &if-merge-request if: '$CI_MERGE_REQUEST_IID' -.if-nightly-master-schedule: &if-nightly-master-schedule - if: '$NIGHTLY && $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule"' +.if-merge-request-title-as-if-foss: &if-merge-request-title-as-if-foss + if: '$CI_MERGE_REQUEST_TITLE =~ /RUN AS-IF-FOSS/' + +.if-security-merge-request: &if-security-merge-request + if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID' .if-dot-com-gitlab-org-schedule: &if-dot-com-gitlab-org-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"' @@ -37,9 +49,6 @@ .if-dot-com-gitlab-org-merge-request: &if-dot-com-gitlab-org-merge-request if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID' -.if-dot-com-gitlab-org-and-security-merge-request: &if-dot-com-gitlab-org-and-security-merge-request - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_MERGE_REQUEST_IID' - .if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_COMMIT_TAG' @@ -52,6 +61,10 @@ #################### # Changes patterns # #################### +.ci-patterns: &ci-patterns + - ".gitlab-ci.yml" + - ".gitlab/ci/**/*" + .yaml-patterns: &yaml-patterns - "**/*.yml" @@ -77,7 +90,6 @@ - "{,ee/}fixtures/**/*" - "{,ee/}rubocop/**/*" - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file .code-patterns: &code-patterns - "{package.json,yarn.lock}" @@ -120,7 +132,6 @@ - "{,ee/}fixtures/**/*" - "{,ee/}rubocop/**/*" - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file .code-qa-patterns: &code-qa-patterns - "{package.json,yarn.lock}" @@ -162,7 +173,6 @@ - "{,ee/}fixtures/**/*" - "{,ee/}rubocop/**/*" - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file # QA changes - ".dockerignore" - "qa/**/*" @@ -173,7 +183,7 @@ .cache-repo:rules: rules: - <<: *if-cache-credentials-schedule - when: on_success + allow_failure: true ############# # CNG rules # @@ -182,6 +192,7 @@ rules: - <<: *if-dot-com-gitlab-org-and-security-tag when: manual + allow_failure: true ###################### # Dev fixtures rules # @@ -208,6 +219,7 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *docs-patterns when: manual + allow_failure: true .docs:rules:docs-lint: rules: @@ -226,6 +238,7 @@ ################## # Frontend rules # ################## +# This job only runs on `master` since it pushes to the cache. .frontend:rules:gitlab-assets-compile-pull-push-cache: rules: - <<: *if-not-canonical-namespace @@ -248,13 +261,14 @@ changes: *code-backstage-qa-patterns when: on_success +# This job only runs on `master` since it pushes to the cache. .frontend:rules:compile-assets-pull-push-cache-as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-master-refs + - <<: *if-master-push changes: *code-backstage-qa-patterns - when: on_success + - <<: *if-master-schedule-2-hourly .frontend:rules:compile-assets-pull-cache: rules: @@ -266,9 +280,14 @@ rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-master-push changes: *code-backstage-qa-patterns - when: on_success + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-backstage-qa-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns .frontend:rules:default-frontend-jobs: rules: @@ -276,13 +295,27 @@ changes: *code-backstage-patterns when: on_success -.frontend:rules:default-frontend-jobs-no-foss: +.frontend:rules:default-frontend-jobs-as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-master-push + changes: *code-backstage-patterns + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns + +.frontend:rules:ee-mr-and-master-only: + rules: + - <<: *if-not-ee + when: never + - <<: *if-merge-request + changes: *code-backstage-patterns + - <<: *if-master-refs changes: *code-backstage-patterns - when: on_success .frontend:rules:qa-frontend-node: rules: @@ -293,6 +326,15 @@ changes: *frontend-dependency-patterns when: on_success +.frontend:rules:qa-frontend-node-latest: + rules: + - <<: *if-master-refs + changes: *frontend-dependency-patterns + allow_failure: true + - <<: *if-merge-request + changes: *frontend-dependency-patterns + allow_failure: true + ################ # Memory rules # ################ @@ -322,24 +364,33 @@ changes: *code-qa-patterns when: on_success -.qa:rules:ee-only: +.qa:rules:as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-master-push changes: *code-qa-patterns - when: on_success + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-qa-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns .qa:rules:package-and-qa: rules: - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns + allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-schedule - when: on_success + allow_failure: true ############### # Rails rules # @@ -348,50 +399,45 @@ rules: - <<: *if-default-refs changes: *code-backstage-patterns - when: on_success .rails:rules:default-refs-code-backstage-qa: rules: - <<: *if-default-refs changes: *code-backstage-qa-patterns - when: on_success - -.rails:rules:master-refs-code-backstage: - rules: - - <<: *if-master-refs - changes: *code-backstage-patterns - when: on_success - - changes: [".gitlab/ci/rails.gitlab-ci.yml"] -.rails:rules:nightly-master-refs-code-backstage: +.rails:rules:ee-only: rules: - - <<: *if-nightly-master-schedule + - <<: *if-not-ee + when: never + - <<: *if-default-refs changes: *code-backstage-patterns - when: on_success - - changes: [".gitlab/ci/rails.gitlab-ci.yml"] -.rails:rules:nightly-master-refs-code-backstage-ee-only: +.rails:rules:as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-nightly-master-schedule + - <<: *if-master-push changes: *code-backstage-patterns - when: on_success - - changes: [".gitlab/ci/rails.gitlab-ci.yml"] + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns -.rails:rules:ee-only: +.rails:rules:ee-mr-and-master-only: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-merge-request + changes: *code-backstage-patterns + - <<: *if-master-refs changes: *code-backstage-patterns - when: on_success .rails:rules:downtime_check: rules: - <<: *if-merge-request changes: *code-backstage-patterns - when: on_success ################## # Releases rules # @@ -414,6 +460,7 @@ # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-patterns + allow_failure: true .reports:rules:sast: rules: @@ -422,6 +469,7 @@ # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns + allow_failure: true .reports:rules:dependency_scanning: rules: @@ -430,6 +478,7 @@ # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns + allow_failure: true .reports:rules:dast: rules: @@ -437,10 +486,11 @@ when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true .reports:schedule-dast: rules: @@ -451,67 +501,86 @@ ################ # Review rules # ################ -.review:rules:mr-and-schedule-auto: +.review:rules:build-qa-image: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: on_success - <<: *if-dot-com-gitlab-org-schedule - when: on_success + +.review:rules:review-build-cng: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *frontend-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise: rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual allow_failure: true - <<: *if-dot-com-gitlab-org-schedule - when: on_success + allow_failure: true -.review:rules:mr-only-auto: +.review:rules:review-stop-failed-deployment: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: on_success -.review:rules:mr-only-auto-if-frontend-manual-otherwise: +.review:rules:review-qa-smoke: rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true .review:rules:mr-only-manual: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true .review:rules:review-cleanup: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-schedule - when: on_success - -.review:rules:review-gcp-cleanup: - rules: - - <<: *if-dot-com-gitlab-org-merge-request - changes: *code-qa-patterns - when: manual - - <<: *if-dot-com-gitlab-org-schedule - when: on_success + allow_failure: true .review:rules:danger: rules: - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID' - when: on_success ############### # Setup rules # @@ -527,10 +596,11 @@ .setup:rules:dont-interrupt-me: rules: - <<: *if-master-or-tag - when: on_success + allow_failure: true - <<: *if-auto-deploy-branches - when: on_success + allow_failure: true - when: manual + allow_failure: true .setup:rules:gitlab_git_test: rules: @@ -557,9 +627,13 @@ .test-metadata:rules:update-tests-metadata: rules: + - <<: *if-not-ee + when: never + - changes: + - ".gitlab/ci/test-metadata.gitlab-ci.yml" + - "scripts/rspec_helpers.sh" - <<: *if-dot-com-ee-schedule changes: *code-backstage-patterns - when: on_success ############## # YAML rules # diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index e79f3939bc7..9be495f1ef2 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -7,7 +7,7 @@ cache gems: - .default-before_script - .setup:rules:cache-gems stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] variables: SETUP_DB: "false" script: @@ -15,6 +15,7 @@ cache gems: artifacts: paths: - vendor/cache + expire_in: 31d .minimal-job: extends: @@ -26,7 +27,6 @@ dont-interrupt-me: stage: sync image: alpine:edge interruptible: false - allow_failure: true variables: GIT_STRATEGY: none script: diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index cda6d996bdb..65cce76fc48 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -31,6 +31,19 @@ update-tests-metadata: - .tests-metadata-state - .test-metadata:rules:update-tests-metadata stage: post-test + dependencies: + - setup-test-env + - rspec migration pg11 + - rspec unit pg11 + - rspec integration pg11 + - rspec system pg11 + - rspec-ee migration pg11 + - rspec-ee unit pg11 + - rspec-ee integration pg11 + - rspec-ee system pg11 + - rspec-ee unit pg11 geo + - rspec-ee integration pg11 geo + - rspec-ee system pg11 geo cache: policy: push script: |