diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
commit | 9f46488805e86b1bc341ea1620b866016c2ce5ed (patch) | |
tree | f9748c7e287041e37d6da49e0a29c9511dc34768 /.gitlab | |
parent | dfc92d081ea0332d69c8aca2f0e745cb48ae5e6d (diff) | |
download | gitlab-ce-9f46488805e86b1bc341ea1620b866016c2ce5ed.tar.gz |
Add latest changes from gitlab-org/gitlab@13-0-stable-ee
Diffstat (limited to '.gitlab')
22 files changed, 392 insertions, 435 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 28ad2b41921..e71e74fd4d3 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -7,8 +7,12 @@ *.rake @gitlab-org/maintainers/rails-backend # Technical writing team are the default reviewers for all markdown docs -*.md @gl-docsteam /doc/ @gl-docsteam +# Dev and Doc guidelines +/doc/development/ @marcia @mjang1 +/doc/development/documentation/ @mikelewis +/doc/ci @marcel.amirault @sselhorn +/doc/.linting @marcel.amirault @eread @aqualls @mikelewis # Frontend maintainers should see everything in `app/assets/` *.scss @annabeldunstone @gitlab-org/maintainers/frontend @@ -33,7 +37,7 @@ /ee/app/finders/ @gitlab-org/maintainers/database # Feature specific owners -/ee/lib/gitlab/code_owners/ @reprazent +/ee/lib/gitlab/code_owners/ @reprazent @kerrizor /ee/lib/ee/gitlab/auth/ldap/ @dblessing @mkozono /lib/gitlab/auth/ldap/ @dblessing @mkozono /lib/gitlab/ci/templates/ @nolith @zj diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml index ecbed0ed6c8..a091785dec3 100644 --- a/.gitlab/ci/cache-repo.gitlab-ci.yml +++ b/.gitlab/ci/cache-repo.gitlab-ci.yml @@ -21,7 +21,6 @@ cache-repo: extends: .cache-repo:rules image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine stage: sync - allow_failure: true variables: GIT_STRATEGY: none TAR_FILENAME: /tmp/gitlab-master.tar diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index 2450e346569..d7699de74e2 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -3,7 +3,6 @@ cloud-native-image: image: ruby:2.6-alpine dependencies: [] stage: post-test - allow_failure: true variables: GIT_DEPTH: "1" script: diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml index 27ceb6f37db..fc3678a7d17 100644 --- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml +++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml @@ -1,11 +1,11 @@ .run-dev-fixtures: extends: - .default-retry - - .default-cache + - .rails-cache - .default-before_script - .use-pg11 stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] variables: FIXTURE_PATH: "db/fixtures/development" SEED_CYCLE_ANALYTICS: "true" @@ -19,8 +19,9 @@ run-dev-fixtures: - .run-dev-fixtures - .dev-fixtures:rules:ee-and-foss script: - - scripts/gitaly-test-spawn - - RAILS_ENV=test bundle exec rake db:seed_fu + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" run-dev-fixtures-ee: extends: @@ -28,6 +29,7 @@ run-dev-fixtures-ee: - .dev-fixtures:rules:ee-only - .use-pg11-ee script: - - scripts/gitaly-test-spawn + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - cp ee/db/fixtures/development/* $FIXTURE_PATH - - RAILS_ENV=test bundle exec rake db:seed_fu + - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index b8a66decbb7..50dbef44598 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -2,7 +2,6 @@ extends: - .default-retry - .docs:rules:review-docs - allow_failure: true image: ruby:2.6-alpine stage: review dependencies: [] @@ -68,7 +67,7 @@ graphql-reference-verify: - .docs:rules:graphql-reference-verify - .use-pg11 stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] script: - bundle exec rake gitlab:graphql:check_docs - bundle exec rake gitlab:graphql:check_schema diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 2b22162b0c2..6e9119f295a 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -15,8 +15,7 @@ - .default-retry - .default-before_script - .assets-compile-cache - - .use-docker-in-docker - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1 stage: prepare variables: NODE_ENV: "production" @@ -34,20 +33,16 @@ paths: - webpack-report/ - assets-compile.log - # We consume these files in GitLab UI for integration tests: - # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1 - - public/assets/application-*.css - - public/assets/application-*.css.gz + # These assets are used in multiple locations: + # - in `build-assets-image` job to create assets image for packaging systems + # - GitLab UI for integration tests: https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1 + - public/assets when: always script: - node --version - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline - free -m - time bin/rake gitlab:assets:compile > assets-compile.log 2>&1 - # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists - # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines - # https://gitlab.com/gitlab-org/gitlab/issues/208389 - - time scripts/build_assets_image - scripts/clean-old-cached-assets - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here @@ -65,6 +60,20 @@ gitlab:assets:compile pull-cache: cache: policy: pull +build-assets-image: + extends: + - .use-kaniko + - .frontend:rules:gitlab-assets-compile-pull-cache + stage: build-images + needs: ["gitlab:assets:compile pull-cache"] + variables: + GIT_DEPTH: "1" + script: + # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists + # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines + # https://gitlab.com/gitlab-org/gitlab/issues/208389 + - scripts/build_assets_image + .compile-assets-metadata: extends: - .default-retry @@ -127,16 +136,15 @@ compile-assets pull-cache as-if-foss: .frontend-fixtures-base: extends: - .default-retry - - .default-cache + - .rails-cache - .default-before_script - .use-pg11 stage: fixtures - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: ["setup-test-env", "compile-assets pull-cache"] script: - - date - - scripts/gitaly-test-spawn - - date - - bundle exec rake frontend:fixtures + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - run_timed_command "bundle exec rake frontend:fixtures" artifacts: name: frontend-fixtures expire_in: 31d @@ -154,7 +162,7 @@ frontend-fixtures: frontend-fixtures-as-if-foss: extends: - .frontend-fixtures-base - - .frontend:rules:default-frontend-jobs-no-foss + - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss .frontend-job-base: @@ -197,7 +205,7 @@ karma: karma-as-if-foss: extends: - .karma-base - - .frontend:rules:default-frontend-jobs-no-foss + - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss needs: ["frontend-fixtures-as-if-foss"] @@ -229,10 +237,24 @@ jest: junit: junit_jest.xml parallel: 2 +jest-integration: + extends: + - .frontend-job-base + - .frontend:rules:default-frontend-jobs + script: + - date + - yarn jest:integration --ci + needs: ["frontend-fixtures"] + cache: + key: jest-integration + paths: + - tmp/cache/jest/ + policy: pull-push + jest-as-if-foss: extends: - .jest-base - - .frontend:rules:default-frontend-jobs-no-foss + - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss needs: ["frontend-fixtures-as-if-foss"] cache: @@ -241,11 +263,13 @@ jest-as-if-foss: coverage-frontend: extends: - .default-retry - - .frontend:rules:default-frontend-jobs-no-foss + - .yarn-cache + - .frontend:rules:ee-mr-and-master-only needs: ["jest"] stage: post-test before_script: - - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline + - source scripts/utils.sh + - retry yarn install --frozen-lockfile script: - yarn node scripts/frontend/merge_coverage_frontend.js artifacts: @@ -254,8 +278,7 @@ coverage-frontend: paths: - coverage-frontend/ cache: - paths: - - .yarn-cache/ + policy: pull .qa-frontend-node: extends: @@ -279,22 +302,32 @@ qa-frontend-node:10: image: node:dubnium qa-frontend-node:latest: - extends: .qa-frontend-node + extends: + - .qa-frontend-node + - .frontend:rules:qa-frontend-node-latest image: node:latest - allow_failure: true webpack-dev-server: extends: - .default-retry - - .default-cache - .frontend:rules:default-frontend-jobs stage: test - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: [] variables: WEBPACK_MEMORY_TEST: "true" WEBPACK_VENDOR_DLL: "true" + cache: + key: + files: + - yarn.lock + prefix: "v1" + paths: + - node_modules/ + - tmp/cache/webpack-dlls/ script: - - yarn webpack-vendor + - source scripts/utils.sh + - retry yarn install --frozen-lockfile + - retry yarn webpack-vendor - node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js artifacts: name: webpack-dev-server diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 66b7c47efcf..e6619ff2b6d 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -21,7 +21,7 @@ # Jobs that only need to pull cache .default-cache: cache: - key: "debian-stretch-ruby-2.6.5-pg11-node-12.x" + key: "debian-stretch-ruby-2.6.6-pg11-node-12.x" paths: - .go/pkg/mod - vendor/ruby @@ -29,65 +29,39 @@ - vendor/gitaly-ruby policy: pull -.use-pg9: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" - services: - - name: postgres:9.6.17 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - variables: - POSTGRES_HOST_AUTH_METHOD: trust +.rails-cache: cache: - key: "debian-stretch-ruby-2.6.5-pg9-node-12.x" + key: + files: + - Gemfile.lock + - GITALY_SERVER_VERSION + prefix: "ruby-go-cache-v1" + paths: + - vendor/ruby + - vendor/gitaly-ruby + - .go/pkg/mod + policy: pull -.use-pg10: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" - services: - - name: postgres:10.12 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - variables: - POSTGRES_HOST_AUTH_METHOD: trust +.yarn-cache: cache: - key: "debian-stretch-ruby-2.6.5-pg10-node-12.x" + key: + files: + - yarn.lock + prefix: "v1" + paths: + - node_modules/ .use-pg11: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine variables: POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg11-node-12.x" - -.use-pg9-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" - services: - - name: postgres:9.6.17 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - - name: elasticsearch:6.4.2 - variables: - POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg9-node-12.x" - -.use-pg10-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" - services: - - name: postgres:10.12 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - - name: elasticsearch:6.4.2 - variables: - POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg10-node-12.x" .use-pg11-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -95,15 +69,13 @@ - name: elasticsearch:6.4.2 variables: POSTGRES_HOST_AUTH_METHOD: trust - cache: - key: "debian-stretch-ruby-2.6.5-pg11-node-12.x" -# Pin kaniko to v0.16.0 due to https://github.com/GoogleContainerTools/kaniko/issues/1162 .use-kaniko: image: - name: gcr.io/kaniko-project/executor:debug-v0.16.0 + name: gcr.io/kaniko-project/executor:debug-v0.20.0 entrypoint: [""] before_script: + - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json .as-if-foss: diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index af75ff257ea..79dfc88d132 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -8,7 +8,7 @@ memory-static: extends: .only-code-memory-job-base stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] variables: SETUP_DB: "false" script: @@ -28,6 +28,7 @@ memory-static: - tmp/memory_*.txt reports: metrics: tmp/memory_metrics.txt + expire_in: 31d # Show memory usage caused by invoking require per gem. # Unlike `memory-static`, it hits the app with one request to ensure that any last minute require-s have been called. @@ -38,7 +39,7 @@ memory-on-boot: - .only-code-memory-job-base - .use-pg11 stage: test - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: ["setup-test-env", "compile-assets pull-cache"] variables: NODE_ENV: "production" RAILS_ENV: "production" @@ -54,3 +55,4 @@ memory-on-boot: - tmp/memory_*.txt reports: metrics: tmp/memory_on_boot_metrics.txt + expire_in: 31d diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index 38d79ddb090..218ec7043d9 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -15,3 +15,4 @@ pages: artifacts: paths: - public + expire_in: 31d diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 8a8f66a4643..40ef13dd92b 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -23,7 +23,7 @@ qa:internal: qa:internal-as-if-foss: extends: - .qa-job-base - - .qa:rules:ee-only + - .qa:rules:as-if-foss - .as-if-foss script: - bundle exec rspec @@ -38,7 +38,7 @@ qa:selectors: qa:selectors-as-if-foss: extends: - qa:selectors - - .qa:rules:ee-only + - .qa:rules:as-if-foss - .as-if-foss .package-and-qa-base: @@ -58,6 +58,5 @@ package-and-qa: needs: - job: build-qa-image artifacts: false - - job: gitlab:assets:compile pull-cache + - job: build-assets-image artifacts: false - allow_failure: true diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 28ec96b838a..e8087aebcef 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,48 +1,43 @@ .rails:needs:setup-and-assets: - needs: ["setup-test-env pg11", "compile-assets pull-cache"] + needs: ["setup-test-env", "compile-assets pull-cache"] .rails-job-base: extends: - .default-retry - - .default-cache - .default-before_script + - .rails-cache -#################### -# EE and FOSS jobs # -.base-setup-test-env: +####################################################### +# EE/FOSS: default refs (MRs, master, schedules) jobs # +setup-test-env: extends: - .rails-job-base + - .rails:rules:default-refs-code-backstage-qa + - .use-pg11 stage: prepare + variables: + GITLAB_TEST_EAGER_LOAD: "0" script: - - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' - - scripts/gitaly-test-build # Do not use 'bundle exec' here + - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'" + - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here + - rm tmp/tests/gitaly/.ruby-bundle # This file prevents gems from being installed even if vendor/gitaly-ruby is missing artifacts: expire_in: 7d paths: - - tmp/tests - config/secrets.yml - - vendor/gitaly-ruby + - tmp/tests/gitaly + - tmp/tests/gitlab-elasticsearch-indexer + - tmp/tests/gitlab-shell + - tmp/tests/gitlab-test-fork + - tmp/tests/gitlab-test-fork_bare + - tmp/tests/gitlab-test + - tmp/tests/gitlab-workhorse + - tmp/tests/repositories + - tmp/tests/second_storage + when: always cache: policy: pull-push -setup-test-env pg11: - extends: - - .base-setup-test-env - - .rails:rules:default-refs-code-backstage-qa - - .use-pg11 - -setup-test-env pg10: - extends: - - .base-setup-test-env - - .rails:rules:master-refs-code-backstage - - .use-pg10 - -setup-test-env pg9: - extends: - - .base-setup-test-env - - .rails:rules:nightly-master-refs-code-backstage - - .use-pg9 - static-analysis: extends: - .rails-job-base @@ -55,7 +50,7 @@ static-analysis: script: - scripts/static-analysis cache: - key: "ruby-2.6.5-pg11-rubocop" + key: "ruby-2.6.6-pg11-rubocop" paths: - vendor/ruby - tmp/rubocop_cache @@ -64,8 +59,8 @@ static-analysis: downtime_check: extends: - .rails-job-base - - .rails:needs:setup-and-assets - .rails:rules:downtime_check + needs: ["setup-test-env"] stage: test variables: SETUP_DB: "false" @@ -75,8 +70,10 @@ downtime_check: .rspec-base: extends: .rails-job-base stage: test - needs: ["setup-test-env pg11", "retrieve-tests-metadata", "compile-assets pull-cache"] + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"] script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" artifacts: @@ -89,7 +86,7 @@ downtime_check: - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ - - junit_rspec.xml + - log/*.log reports: junit: junit_rspec.xml @@ -101,6 +98,8 @@ downtime_check: .rspec-base-migration: script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration" @@ -133,7 +132,7 @@ rspec fast_spec_helper: - .rails:rules:ee-and-foss - .use-pg11 stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] db:migrate:reset: extends: .db-job-base @@ -141,28 +140,28 @@ db:migrate:reset: - bundle exec rake db:migrate:reset db:check-schema: - extends: .db-job-base + extends: + - .db-job-base + - .rails:rules:ee-mr-and-master-only script: - source scripts/schema_changed.sh -db:migrate-from-v11.11.0: +db:migrate-from-v12.10.0: extends: .db-job-base variables: SETUP_DB: "false" script: - - export PROJECT_TO_CHECKOUT="gitlab-foss" - - export TAG_TO_CHECKOUT="v11.11.0" - - '[[ ! -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"' - - '[[ ! -d "ee/" ]] || export TAG_TO_CHECKOUT="v11.11.0-ee"' + - export PROJECT_TO_CHECKOUT="gitlab" + - export TAG_TO_CHECKOUT="v12.10.0-ee" + - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab-foss"' + - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="v12.10.0"' - git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT - git checkout -f FETCH_HEAD - - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile - - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile - bundle update google-protobuf grpc bootsnap - bundle install $BUNDLE_INSTALL_FLAGS - date - cp config/gitlab.yml.example config/gitlab.yml - - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - bundle exec rake db:drop db:create db:structure:load db:seed_fu - date - git checkout -f $CI_COMMIT_SHA - bundle install $BUNDLE_INSTALL_FLAGS @@ -186,23 +185,24 @@ gitlab:setup: # db/fixtures/development/04_project.rb thanks to SIZE=1 below - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git - - scripts/gitaly-test-spawn + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup artifacts: when: on_failure expire_in: 1d paths: - - log/development.log + - log/*.log rspec:coverage: extends: - .rails-job-base - - .rails:rules:ee-only + - .rails:rules:ee-mr-and-master-only stage: post-test # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) # so we use `dependencies` here. dependencies: - - setup-test-env pg11 + - setup-test-env - rspec migration pg11 - rspec unit pg11 - rspec integration pg11 @@ -231,104 +231,11 @@ rspec:coverage: - coverage/index.html - coverage/assets/ - tmp/memory_test/ -# EE and FOSS jobs # -#################### - -#################### -# master-only jobs # -.rspec-base-pg10: - extends: - - .rspec-base - - .rails:rules:master-refs-code-backstage - - .use-pg10 - needs: ["setup-test-env pg10", "retrieve-tests-metadata", "compile-assets pull-cache"] +# EE/FOSS: default refs (MRs, master, schedules) jobs # +####################################################### -rspec migration pg10: - extends: - - .rspec-base-pg10 - - .rspec-base-migration - parallel: 5 - -rspec unit pg10: - extends: .rspec-base-pg10 - parallel: 20 - -rspec integration pg10: - extends: .rspec-base-pg10 - parallel: 8 - -rspec system pg10: - extends: .rspec-base-pg10 - parallel: 24 -# master-only jobs # -#################### - -###################### -# nightly-only jobs # -.rspec-base-pg9: - extends: - - .rspec-base - - .rails:rules:nightly-master-refs-code-backstage - - .use-pg9 - needs: ["setup-test-env pg9", "retrieve-tests-metadata", "compile-assets pull-cache"] - -rspec migration pg9: - extends: - - .rspec-base-pg9 - - .rspec-base-migration - parallel: 5 - -rspec unit pg9: - extends: .rspec-base-pg9 - parallel: 20 - -rspec integration pg9: - extends: .rspec-base-pg9 - parallel: 8 - -rspec system pg9: - extends: .rspec-base-pg9 - parallel: 24 -# nightly-only jobs # -##################### - -####################### -# EE master-only jobs # -.rspec-ee-base-pg10: - extends: - - .rspec-base-ee - - .use-pg10-ee - needs: ["setup-test-env pg10", "retrieve-tests-metadata", "compile-assets pull-cache"] - -rspec-ee migration pg10: - extends: - - .rspec-ee-base-pg10 - - .rspec-base-migration - - .rails:rules:master-refs-code-backstage - parallel: 2 - -rspec-ee unit pg10: - extends: - - .rspec-ee-base-pg10 - - .rails:rules:master-refs-code-backstage - parallel: 10 - -rspec-ee integration pg10: - extends: - - .rspec-ee-base-pg10 - - .rails:rules:master-refs-code-backstage - parallel: 4 - -rspec-ee system pg10: - extends: - - .rspec-ee-base-pg10 - - .rails:rules:master-refs-code-backstage - parallel: 6 -# EE master-only jobs # -####################### - -################ -# EE-only jobs # +################################################## +# EE: default refs (MRs, master, schedules) jobs # .rspec-base-ee: extends: - .rspec-base @@ -336,10 +243,11 @@ rspec-ee system pg10: .rspec-base-pg11-as-if-foss: extends: - - .rspec-base-ee + - .rspec-base + - .rails:rules:as-if-foss - .as-if-foss - .use-pg11 - needs: ["setup-test-env pg11", "retrieve-tests-metadata", "compile-assets pull-cache as-if-foss"] + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache as-if-foss"] .rspec-ee-base-pg11: extends: @@ -385,6 +293,8 @@ rspec-ee system pg11: .rspec-ee-base-geo: extends: .rspec-base-ee script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - scripts/prepare_postgres_fdw.sh - rspec_paralellized_job "--tag ~quarantine --tag geo" @@ -411,26 +321,5 @@ db:rollback geo: script: - bundle exec rake geo:db:migrate VERSION=20170627195211 - bundle exec rake geo:db:migrate -# EE-only jobs # -################ - -######################## -# EE nightly-only jobs # -.rspec-ee-base-geo-pg9: - extends: - - .rspec-ee-base-geo - - .use-pg9-ee - - .rails:rules:nightly-master-refs-code-backstage-ee-only - needs: ["setup-test-env pg9", "retrieve-tests-metadata", "compile-assets pull-cache"] - -rspec-ee unit pg9 geo: - extends: .rspec-ee-base-geo-pg9 - parallel: 2 - -rspec-ee integration pg9 geo: - extends: .rspec-ee-base-geo-pg9 - -rspec-ee system pg9 geo: - extends: .rspec-ee-base-geo-pg9 -# EE nightly-only jobs # -######################## +# EE: default refs (MRs, master, schedules) jobs # +################################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 61915aa798e..153334e1aff 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -14,11 +14,7 @@ code_quality: - .use-docker-in-docker stage: test needs: [] - allow_failure: true variables: - # emptying DOCKER_HOST so it can be detected properly on kubernetes executor - # with the script below - DOCKER_HOST: "" CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.9" script: - | @@ -47,12 +43,10 @@ code_quality: extends: - .default-retry - .reports:rules:sast - - .use-docker-in-docker stage: test # `needs: []` starts the job immediately in the pipeline # https://docs.gitlab.com/ee/ci/yaml/README.html#needs needs: [] - allow_failure: true artifacts: paths: - gl-sast-report.json # GitLab-specific @@ -60,10 +54,6 @@ code_quality: sast: gl-sast-report.json expire_in: 1 week # GitLab-specific variables: - # emptying DOCKER_HOST so it can be detected properly on kubernetes executor - # with the script below - DOCKER_HOST: "" - DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SAST_ANALYZER_IMAGE_TAG: 2 @@ -82,15 +72,11 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" -kubesec-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" - -nodejs-scan-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" +# Temporary disabled as it's constantly failing. See https://gitlab.com/gitlab-org/gitlab/-/issues/213769. +# nodejs-scan-sast: +# extends: .sast +# image: +# name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" secrets-sast: extends: .sast @@ -108,11 +94,7 @@ dependency_scanning: stage: test needs: [] variables: - # emptying DOCKER_HOST so it can be detected properly on kubernetes executor - # with the script below - DOCKER_HOST: "" DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec" # GitLab-specific - allow_failure: true script: - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | @@ -183,7 +165,6 @@ dast: # DAST_USERNAME_FIELD: "user[login]" # DAST_PASSWORD_FIELD: "user[passowrd]" DAST_VERSION: 1 - allow_failure: true script: - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' # To be done in a later iteration diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 06710b3b9dd..46a281cd48f 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,70 +1,36 @@ -.review-docker: - extends: - - .default-retry - - .use-docker-in-docker - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6 - variables: - GITLAB_EDITION: "ce" - build-qa-image: extends: - .use-kaniko - .default-retry - - .review:rules:mr-and-schedule-auto - stage: prepare + - .review:rules:build-qa-image + stage: build-images + needs: [] script: - - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true -.review-cleanup-base: +review-cleanup: extends: - .default-retry - .review:rules:review-cleanup + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 stage: prepare - allow_failure: true environment: name: review/auto-cleanup action: stop before_script: - source scripts/utils.sh + - source scripts/review_apps/gcp_cleanup.sh - install_gitlab_gem + - setup_gcp_dependencies script: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb - -review-cleanup: - extends: - - .review-cleanup-base - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - -review-cleanup-helm3: - extends: - - .review-cleanup-base - variables: - HELM_3: 1 - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 - -review-gcp-cleanup: - extends: - - .review:rules:review-gcp-cleanup - stage: prepare - image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest - allow_failure: true - environment: - name: review/auto-gcp-cleanup - action: stop - before_script: - - gcloud auth activate-service-account --key-file=$REVIEW_APPS_GCP_CREDENTIALS - - gcloud config set project $REVIEW_APPS_GCP_PROJECT - - apt-get install -y jq - - source scripts/review_apps/gcp_cleanup.sh - script: - gcp_cleanup review-build-cng: extends: - .default-retry - - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + - .review:rules:review-build-cng image: ruby:2.6-alpine stage: review-prepare before_script: @@ -87,8 +53,7 @@ review-build-cng: variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v3.2.2" - GITLAB_EDITION: "ce" + GITLAB_HELM_CHART_REF: "v3.3.3" environment: name: review/${CI_COMMIT_REF_NAME} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} @@ -102,9 +67,7 @@ review-deploy: stage: review dependencies: [] resource_group: "review/${CI_COMMIT_REF_NAME}" - allow_failure: true before_script: - - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) @@ -146,28 +109,28 @@ review-deploy: review-stop-failed-deployment: extends: - .review-stop-base - - .review:rules:mr-only-auto + - .review:rules:review-stop-failed-deployment stage: prepare script: - delete_failed_release - - delete_helm2_release review-stop: extends: - .review-stop-base - .review:rules:mr-only-manual stage: review - allow_failure: true script: - delete_release .review-qa-base: - extends: .review-docker + extends: + - .default-retry + - .use-docker-in-docker + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6 stage: qa # This is needed so that manual jobs with needs don't block the pipeline. # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. dependencies: ["review-deploy"] - allow_failure: true variables: QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" QA_CAN_TEST_GIT_PROTOCOL_V2: "false" @@ -179,8 +142,7 @@ review-stop: GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" before_script: - - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - echo "${QA_IMAGE}" @@ -196,7 +158,7 @@ review-stop: review-qa-smoke: extends: - .review-qa-base - - .review:rules:mr-only-auto-if-frontend-manual-otherwise + - .review:rules:review-qa-smoke script: - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" @@ -221,7 +183,6 @@ review-performance: # This is needed so that manual jobs with needs don't block the pipeline. # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. dependencies: ["review-deploy"] - allow_failure: true before_script: - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" @@ -237,6 +198,7 @@ review-performance: - sitespeed-results/ reports: performance: performance.json + expire_in: 31d parallel-spec-reports: extends: @@ -244,7 +206,6 @@ parallel-spec-reports: image: ruby:2.6-alpine stage: post-qa dependencies: ["review-qa-all"] - allow_failure: true variables: NEW_PARALLEL_SPECS_REPORT: qa/report-new.html BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" @@ -263,17 +224,19 @@ parallel-spec-reports: - qa/gitlab-qa-run-* reports: junit: qa/gitlab-qa-run-*/**/rspec-*.xml + expire_in: 31d danger-review: extends: - .default-retry - - .default-cache + - .yarn-cache - .review:rules:danger image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test needs: [] script: - - git version - - node --version - - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline + - source scripts/utils.sh + - retry yarn install --frozen-lockfile - danger --fail-on-errors=true --verbose + cache: + policy: pull diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 79ae7823853..383aca0043b 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -16,6 +16,15 @@ .if-master-refs: &if-master-refs if: '$CI_COMMIT_REF_NAME == "master"' +.if-master-push: &if-master-push + if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "push"' + +.if-master-schedule-2-hourly: &if-master-schedule-2-hourly + if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' + +.if-master-schedule-nightly: &if-master-schedule-nightly + if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "nightly"' + .if-auto-deploy-branches: &if-auto-deploy-branches if: '$CI_COMMIT_BRANCH =~ /^\d+-\d+-auto-deploy-\d+$/' @@ -25,8 +34,11 @@ .if-merge-request: &if-merge-request if: '$CI_MERGE_REQUEST_IID' -.if-nightly-master-schedule: &if-nightly-master-schedule - if: '$NIGHTLY && $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule"' +.if-merge-request-title-as-if-foss: &if-merge-request-title-as-if-foss + if: '$CI_MERGE_REQUEST_TITLE =~ /RUN AS-IF-FOSS/' + +.if-security-merge-request: &if-security-merge-request + if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID' .if-dot-com-gitlab-org-schedule: &if-dot-com-gitlab-org-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"' @@ -37,9 +49,6 @@ .if-dot-com-gitlab-org-merge-request: &if-dot-com-gitlab-org-merge-request if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID' -.if-dot-com-gitlab-org-and-security-merge-request: &if-dot-com-gitlab-org-and-security-merge-request - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_MERGE_REQUEST_IID' - .if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_COMMIT_TAG' @@ -52,6 +61,10 @@ #################### # Changes patterns # #################### +.ci-patterns: &ci-patterns + - ".gitlab-ci.yml" + - ".gitlab/ci/**/*" + .yaml-patterns: &yaml-patterns - "**/*.yml" @@ -77,7 +90,6 @@ - "{,ee/}fixtures/**/*" - "{,ee/}rubocop/**/*" - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file .code-patterns: &code-patterns - "{package.json,yarn.lock}" @@ -120,7 +132,6 @@ - "{,ee/}fixtures/**/*" - "{,ee/}rubocop/**/*" - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file .code-qa-patterns: &code-qa-patterns - "{package.json,yarn.lock}" @@ -162,7 +173,6 @@ - "{,ee/}fixtures/**/*" - "{,ee/}rubocop/**/*" - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file # QA changes - ".dockerignore" - "qa/**/*" @@ -173,7 +183,7 @@ .cache-repo:rules: rules: - <<: *if-cache-credentials-schedule - when: on_success + allow_failure: true ############# # CNG rules # @@ -182,6 +192,7 @@ rules: - <<: *if-dot-com-gitlab-org-and-security-tag when: manual + allow_failure: true ###################### # Dev fixtures rules # @@ -208,6 +219,7 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *docs-patterns when: manual + allow_failure: true .docs:rules:docs-lint: rules: @@ -226,6 +238,7 @@ ################## # Frontend rules # ################## +# This job only runs on `master` since it pushes to the cache. .frontend:rules:gitlab-assets-compile-pull-push-cache: rules: - <<: *if-not-canonical-namespace @@ -248,13 +261,14 @@ changes: *code-backstage-qa-patterns when: on_success +# This job only runs on `master` since it pushes to the cache. .frontend:rules:compile-assets-pull-push-cache-as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-master-refs + - <<: *if-master-push changes: *code-backstage-qa-patterns - when: on_success + - <<: *if-master-schedule-2-hourly .frontend:rules:compile-assets-pull-cache: rules: @@ -266,9 +280,14 @@ rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-master-push changes: *code-backstage-qa-patterns - when: on_success + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-backstage-qa-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns .frontend:rules:default-frontend-jobs: rules: @@ -276,13 +295,27 @@ changes: *code-backstage-patterns when: on_success -.frontend:rules:default-frontend-jobs-no-foss: +.frontend:rules:default-frontend-jobs-as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-master-push + changes: *code-backstage-patterns + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns + +.frontend:rules:ee-mr-and-master-only: + rules: + - <<: *if-not-ee + when: never + - <<: *if-merge-request + changes: *code-backstage-patterns + - <<: *if-master-refs changes: *code-backstage-patterns - when: on_success .frontend:rules:qa-frontend-node: rules: @@ -293,6 +326,15 @@ changes: *frontend-dependency-patterns when: on_success +.frontend:rules:qa-frontend-node-latest: + rules: + - <<: *if-master-refs + changes: *frontend-dependency-patterns + allow_failure: true + - <<: *if-merge-request + changes: *frontend-dependency-patterns + allow_failure: true + ################ # Memory rules # ################ @@ -322,24 +364,33 @@ changes: *code-qa-patterns when: on_success -.qa:rules:ee-only: +.qa:rules:as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-master-push changes: *code-qa-patterns - when: on_success + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-qa-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns .qa:rules:package-and-qa: rules: - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns + allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-schedule - when: on_success + allow_failure: true ############### # Rails rules # @@ -348,50 +399,45 @@ rules: - <<: *if-default-refs changes: *code-backstage-patterns - when: on_success .rails:rules:default-refs-code-backstage-qa: rules: - <<: *if-default-refs changes: *code-backstage-qa-patterns - when: on_success - -.rails:rules:master-refs-code-backstage: - rules: - - <<: *if-master-refs - changes: *code-backstage-patterns - when: on_success - - changes: [".gitlab/ci/rails.gitlab-ci.yml"] -.rails:rules:nightly-master-refs-code-backstage: +.rails:rules:ee-only: rules: - - <<: *if-nightly-master-schedule + - <<: *if-not-ee + when: never + - <<: *if-default-refs changes: *code-backstage-patterns - when: on_success - - changes: [".gitlab/ci/rails.gitlab-ci.yml"] -.rails:rules:nightly-master-refs-code-backstage-ee-only: +.rails:rules:as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-nightly-master-schedule + - <<: *if-master-push changes: *code-backstage-patterns - when: on_success - - changes: [".gitlab/ci/rails.gitlab-ci.yml"] + - <<: *if-master-schedule-2-hourly + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns -.rails:rules:ee-only: +.rails:rules:ee-mr-and-master-only: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-merge-request + changes: *code-backstage-patterns + - <<: *if-master-refs changes: *code-backstage-patterns - when: on_success .rails:rules:downtime_check: rules: - <<: *if-merge-request changes: *code-backstage-patterns - when: on_success ################## # Releases rules # @@ -414,6 +460,7 @@ # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-patterns + allow_failure: true .reports:rules:sast: rules: @@ -422,6 +469,7 @@ # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns + allow_failure: true .reports:rules:dependency_scanning: rules: @@ -430,6 +478,7 @@ # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns + allow_failure: true .reports:rules:dast: rules: @@ -437,10 +486,11 @@ when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true .reports:schedule-dast: rules: @@ -451,67 +501,86 @@ ################ # Review rules # ################ -.review:rules:mr-and-schedule-auto: +.review:rules:build-qa-image: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: on_success - <<: *if-dot-com-gitlab-org-schedule - when: on_success + +.review:rules:review-build-cng: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *frontend-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise: rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual allow_failure: true - <<: *if-dot-com-gitlab-org-schedule - when: on_success + allow_failure: true -.review:rules:mr-only-auto: +.review:rules:review-stop-failed-deployment: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: on_success -.review:rules:mr-only-auto-if-frontend-manual-otherwise: +.review:rules:review-qa-smoke: rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: on_success + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true .review:rules:mr-only-manual: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true .review:rules:review-cleanup: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-schedule - when: on_success - -.review:rules:review-gcp-cleanup: - rules: - - <<: *if-dot-com-gitlab-org-merge-request - changes: *code-qa-patterns - when: manual - - <<: *if-dot-com-gitlab-org-schedule - when: on_success + allow_failure: true .review:rules:danger: rules: - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID' - when: on_success ############### # Setup rules # @@ -527,10 +596,11 @@ .setup:rules:dont-interrupt-me: rules: - <<: *if-master-or-tag - when: on_success + allow_failure: true - <<: *if-auto-deploy-branches - when: on_success + allow_failure: true - when: manual + allow_failure: true .setup:rules:gitlab_git_test: rules: @@ -557,9 +627,13 @@ .test-metadata:rules:update-tests-metadata: rules: + - <<: *if-not-ee + when: never + - changes: + - ".gitlab/ci/test-metadata.gitlab-ci.yml" + - "scripts/rspec_helpers.sh" - <<: *if-dot-com-ee-schedule changes: *code-backstage-patterns - when: on_success ############## # YAML rules # diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index e79f3939bc7..9be495f1ef2 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -7,7 +7,7 @@ cache gems: - .default-before_script - .setup:rules:cache-gems stage: test - needs: ["setup-test-env pg11"] + needs: ["setup-test-env"] variables: SETUP_DB: "false" script: @@ -15,6 +15,7 @@ cache gems: artifacts: paths: - vendor/cache + expire_in: 31d .minimal-job: extends: @@ -26,7 +27,6 @@ dont-interrupt-me: stage: sync image: alpine:edge interruptible: false - allow_failure: true variables: GIT_STRATEGY: none script: diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index cda6d996bdb..65cce76fc48 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -31,6 +31,19 @@ update-tests-metadata: - .tests-metadata-state - .test-metadata:rules:update-tests-metadata stage: post-test + dependencies: + - setup-test-env + - rspec migration pg11 + - rspec unit pg11 + - rspec integration pg11 + - rspec system pg11 + - rspec-ee migration pg11 + - rspec-ee unit pg11 + - rspec-ee integration pg11 + - rspec-ee system pg11 + - rspec-ee unit pg11 geo + - rspec-ee integration pg11 geo + - rspec-ee system pg11 geo cache: policy: push script: diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index 4b98b4e7ac6..45b5fc85cd1 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -1,4 +1,4 @@ -<!-- The first three sections: "Problem to solve", "Intended users" and "Proposal", are strongly recommended, while the rest of the sections can be filled out during the problem validation or breakdown phase. However, keep in mind that providing complete and relevant information early helps our product team validate the problem and start working on a solution. --> +<!-- The first four sections: "Problem to solve", "Intended users", "User experience goal", and "Proposal", are strongly recommended, while the rest of the sections can be filled out during the problem validation or breakdown phase. However, keep in mind that providing complete and relevant information early helps our product team validate the problem and start working on a solution. --> ### Problem to solve @@ -10,33 +10,44 @@ Personas are described at https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/ -* [Rachel (Release Manager)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#rachel-release-manager) +* [Cameron (Compliance Manager)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#cameron-compliance-manager) * [Parker (Product Manager)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#parker-product-manager) * [Delaney (Development Team Lead)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#delaney-development-team-lead) -* [Sasha (Software Developer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sasha-software-developer) * [Presley (Product Designer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#presley-product-designer) +* [Sasha (Software Developer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sasha-software-developer) * [Devon (DevOps Engineer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#devon-devops-engineer) * [Sidney (Systems Administrator)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sidney-systems-administrator) * [Sam (Security Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sam-security-analyst) -* [Dana (Data Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#dana-data-analyst) +* [Rachel (Release Manager)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#rachel-release-manager) +* [Alex (Security Operations Engineer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#alex-security-operations-engineer) * [Simone (Software Engineer in Test)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#simone-software-engineer-in-test) -* [Allison (Application Ops)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#allison-application-ops) --> +* [Allison (Application Ops)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#allison-application-ops) +* [Priyanka (Platform Engineer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#priyanka-platform-engineer) +* [Dana (Data Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#dana-data-analyst) +--> -### Further details +### User experience goal + +<!-- What is the single user experience workflow this problem addresses? +For example, "The user should be able to use the UI/API/.gitlab-ci.yml with GitLab to <perform a specific task>" +https://about.gitlab.com/handbook/engineering/ux/ux-research-training/user-story-mapping/ --> -<!-- Include use cases, benefits, goals, or any other details that will help us understand the problem better. --> ### Proposal <!-- How are we going to solve the problem? Try to include the user journey! https://about.gitlab.com/handbook/journeys/#user-journey --> +### Further details + +<!-- Include use cases, benefits, goals, or any other details that will help us understand the problem better. --> + ### Permissions and Security <!-- What permissions are required to perform the described actions? Are they consistent with the existing permissions as documented for users, groups, and projects as appropriate? Is the proposed behavior consistent between the UI, API, and other access methods (e.g. email replies)?--> ### Documentation -<!-- See the Feature Change Documentation Workflow https://docs.gitlab.com/ee/development/documentation/feature-change-workflow.html +<!-- See the Feature Change Documentation Workflow https://docs.gitlab.com/ee/development/documentation/workflow.html#for-a-product-change * Add all known Documentation Requirements in this section. See https://docs.gitlab.com/ee/development/documentation/feature-change-workflow.html#documentation-requirements * If this feature requires changing permissions, update the permissions document. See https://docs.gitlab.com/ee/user/permissions.html --> diff --git a/.gitlab/issue_templates/Productivity Improvement.md b/.gitlab/issue_templates/Productivity Improvement.md index 79e1461392e..974f11f6da3 100644 --- a/.gitlab/issue_templates/Productivity Improvement.md +++ b/.gitlab/issue_templates/Productivity Improvement.md @@ -1,7 +1,7 @@ -## What is the productivity problem to solve? +## What is the GitLab engineering productivity problem to solve? <!-- -Please describe the productivity problem that needs to be solved backed by charts from +Please describe the engineering productivity problem that needs to be solved backed by charts from https://about.gitlab.com/handbook/engineering/quality/engineering-productivity-team/#engineering-productivity-team-metrics. --> @@ -37,4 +37,3 @@ after the implementation is merged/deployed/released. - Otherwise, create a new "Productivity Improvement" issue. You can re-use the description from this issue, but obviously another solution should be chosen this time. /label ~"Engineering Productivity" ~meta -/cc @gl-quality/eng-prod diff --git a/.gitlab/issue_templates/Security developer workflow.md b/.gitlab/issue_templates/Security developer workflow.md index 2c80360d32d..695f0167ad4 100644 --- a/.gitlab/issue_templates/Security developer workflow.md +++ b/.gitlab/issue_templates/Security developer workflow.md @@ -36,7 +36,8 @@ After your merge request has been approved according to our [approval guidelines ## Documentation and final details - [ ] Ensure the [Links section](#links) is completed. -- [ ] Find out the versions affected (the Git history of the files affected may help you with this) and add them to the [details section](#details) +- [ ] Add the GitLab [versions](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/security/developer.md#versions-affected) and editions affected to the [details section](#details) + * The Git history of the files affected may help you associate the issue with a [release](https://about.gitlab.com/releases/) - [ ] Fill in any upgrade notes that users may need to take into account in the [details section](#details) - [ ] Add Yes/No and further details if needed to the migration and settings columns in the [details section](#details) - [ ] Add the nickname of the external user who found the issue (and/or HackerOne profile) to the Thanks row in the [details section](#details) @@ -56,6 +57,7 @@ After your merge request has been approved according to our [approval guidelines | Description | Details | Further details| | -------- | -------- | -------- | | Versions affected | X.Y | | +| GitLab EE only | Yes/No | | | Upgrade notes | | | | GitLab Settings updated | Yes/No| | | Migration required | Yes/No | | diff --git a/.gitlab/issue_templates/Technical Evaluation.md b/.gitlab/issue_templates/Technical Evaluation.md index f603d88a764..533a1343820 100644 --- a/.gitlab/issue_templates/Technical Evaluation.md +++ b/.gitlab/issue_templates/Technical Evaluation.md @@ -7,7 +7,7 @@ ### Tasks to Evaluate -<!-- Outline the tasks with issues that you need evaluate as a part of the implementation issue --> +<!-- Outline the tasks with issues that you need to evaluate as a part of the implementation issue --> - [ ] Determine feasibility of the feature - [ ] Create issue for implementation or update existing implementation issue description with implementation proposal @@ -18,7 +18,7 @@ ### Risks and Implementation Considerations -<!-- Idenitfy any risks found in the research, whether this is performance, impacts to other functionality or other bugs --> +<!-- Identify any risks found in the research, whether this is performance, impacts to other functionality or other bugs --> ### Team diff --git a/.gitlab/merge_request_templates/Documentation.md b/.gitlab/merge_request_templates/Documentation.md index 901228ee77e..72bfd2cdec4 100644 --- a/.gitlab/merge_request_templates/Documentation.md +++ b/.gitlab/merge_request_templates/Documentation.md @@ -13,12 +13,25 @@ <!-- Link related issues below. Insert the issue link or reference after the word "Closes" if merging this should automatically close it. --> -## Author's checklist +## Author's checklist (required) - [ ] Follow the [Documentation Guidelines](https://docs.gitlab.com/ee/development/documentation/) and [Style Guide](https://docs.gitlab.com/ee/development/documentation/styleguide.html). -- [ ] If applicable, update the [permissions table](https://docs.gitlab.com/ee/user/permissions.html). +- If you have `developer` access or higher (for example, GitLab team members or [Core Team](https://about.gitlab.com/community/core-team/) members) + - [ ] Apply the ~documentation label, plus: + - The corresponding DevOps stage and group label, if applicable. + - ~"development guidelines" when changing docs under `doc/development/*`, `CONTRIBUTING.md`, or `README.md`. + - ~"development guidelines" and ~"Documentation guidelines" when changing docs under `development/documentation/*`. + - ~"development guidelines" and ~"Description templates (.gitlab/\*)" when creating/updating issue and MR description templates. + - [ ] Assign the [designated Technical Writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments). + +When applicable: + +- [ ] Update the [permissions table](https://docs.gitlab.com/ee/user/permissions.html). - [ ] Link docs to and from the higher-level index page, plus other related docs where helpful. -- [ ] Apply the ~documentation label. +- [ ] Add [GitLab's version history note(s)](https://docs.gitlab.com/ee/development/documentation/styleguide.html#text-for-documentation-requiring-version-text). +- [ ] Add the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide.html#product-badges). +- [ ] Add/update the [feature flag section](https://docs.gitlab.com/ee/development/documentation/feature_flags.html). +- [ ] If you're changing document headings, search `doc/*`, `app/views/*`, and `ee/app/views/*` for old headings replacing with the new ones to [avoid broken anchors](https://docs.gitlab.com/ee/development/documentation/styleguide.html#anchor-links). ## Review checklist @@ -30,7 +43,9 @@ All reviewers can help ensure accuracy, clarity, completeness, and adherence to **2. Technical Writer** -* [ ] Optional: Technical writer review. If not requested for this MR, must be scheduled post-merge. To request for this MR, assign the writer listed for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). +- [ ] Optional: Technical writer review. If not requested for this MR, must be scheduled post-merge. To request for this MR, assign the writer listed for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). + - [ ] Add ~"Technical Writing" and `docs::` workflow label. + - [ ] Add ~docs-only when the only files changed are under `doc/*`. **3. Maintainer** diff --git a/.gitlab/merge_request_templates/Security Release.md b/.gitlab/merge_request_templates/Security Release.md index af3839a96a4..f852bebae95 100644 --- a/.gitlab/merge_request_templates/Security Release.md +++ b/.gitlab/merge_request_templates/Security Release.md @@ -19,7 +19,7 @@ See [the general developer security release guidelines](https://gitlab.com/gitla - [ ] A [CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html) is added without a `merge_request` value, with `type` set to `security` - [ ] Assign to a reviewer and maintainer, per our [Code Review process]. - [ ] For the MR targeting `master`: - - [ ] Ping appsec team member who created the issue and ask for a non-blocking review with `Please review this MR`. + - [ ] Ask for a non-blocking review from the AppSec team member associated to the issue in the [Canonical repository](https://gitlab.com/gitlab-org/gitlab). If you're unsure who to ping, ask on `#sec-appsec` Slack channel. - [ ] Ensure it's approved according to our [Approval Guidelines]. - [ ] Merge request _must not_ close the corresponding security issue, _unless_ it targets `master`. |