diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
commit | 311b0269b4eb9839fa63f80c8d7a58f32b8138a0 (patch) | |
tree | 07e7870bca8aed6d61fdcc810731c50d2c40af47 /.gitlab/ci | |
parent | 27909cef6c4170ed9205afa7426b8d3de47cbb0c (diff) | |
download | gitlab-ce-311b0269b4eb9839fa63f80c8d7a58f32b8138a0.tar.gz |
Add latest changes from gitlab-org/gitlab@14-5-stable-eev14.5.0-rc42
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/cache-repo.gitlab-ci.yml | 63 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 24 | ||||
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 14 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 30 | ||||
-rw-r--r-- | .gitlab/ci/qa-report.gitlab-ci.yml | 15 | ||||
-rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 54 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 280 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 12 | ||||
-rw-r--r-- | .gitlab/ci/review-apps/main.gitlab-ci.yml | 7 | ||||
-rw-r--r-- | .gitlab/ci/review-apps/qa.gitlab-ci.yml | 40 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 244 | ||||
-rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 32 | ||||
-rw-r--r-- | .gitlab/ci/static-analysis.gitlab-ci.yml | 92 | ||||
-rw-r--r-- | .gitlab/ci/workhorse.gitlab-ci.yml | 15 | ||||
-rw-r--r-- | .gitlab/ci/yaml.gitlab-ci.yml | 2 |
15 files changed, 711 insertions, 213 deletions
diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml deleted file mode 100644 index 98c8c72ae3a..00000000000 --- a/.gitlab/ci/cache-repo.gitlab-ci.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Builds a cached .tar.gz of the $CI_DEFAULT_BRANCH branch with full history and -# uploads it to Google Cloud Storage. This archive is downloaded by a -# script defined by a CI/CD variable named CI_PRE_CLONE_SCRIPT. This has -# two benefits: -# -# 1. It speeds up builds. A 800 MB download only takes seconds. -# 2. It significantly reduces load on the file server. Smaller deltas -# means less time spent in git pack-objects. -# -# Since the destination directory of the archive depends on the project -# ID, this is only run on GitLab.com. -# -# CI_REPO_CACHE_CREDENTIALS contains the Google Cloud service account -# JSON for uploading to the gitlab-ci-git-repo-cache bucket. These -# credentials are stored in the Production vault. -# -# Note that this bucket should be located in the same continent as the -# runner, or network egress charges will apply: -# https://cloud.google.com/storage/pricing -cache-repo: - extends: .cache-repo:rules - image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine - stage: sync - variables: - GIT_STRATEGY: none - SHALLOW_CLONE_TAR_FILENAME: gitlab-master-shallow.tar - FULL_CLONE_TAR_FILENAME: gitlab-master.tar - before_script: - - '[ -z "$CI_REPO_CACHE_CREDENTIALS" ] || gcloud auth activate-service-account --key-file=$CI_REPO_CACHE_CREDENTIALS' - script: - # Enable shallow repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well) - # The `git repack` call works around a Git bug with shallow clones: https://gitlab.com/gitlab-org/git/-/issues/86 - - if [ -z "$DISABLE_SHALLOW_REPO_CACHING" ]; then - cd .. && rm -rf $CI_PROJECT_NAME; - today=$(date +%Y-%m-%d); - year=$(date +%Y); - last_year=`expr $year - 1`; - one_year_ago=$(echo $today | sed "s/$year/$last_year/"); - echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME with commits from $one_year_ago."; - time git clone --progress --no-checkout --shallow-since=$one_year_ago $CI_REPOSITORY_URL $CI_PROJECT_NAME; - cd $CI_PROJECT_NAME; - time git repack -d; - echo "Archiving $CI_PROJECT_NAME into /tmp/$SHALLOW_CLONE_TAR_FILENAME."; - time git remote rm origin; - time tar cf /tmp/$SHALLOW_CLONE_TAR_FILENAME .; - echo "GZipping /tmp/$SHALLOW_CLONE_TAR_FILENAME."; - time gzip /tmp/$SHALLOW_CLONE_TAR_FILENAME; - [ -z "$CI_REPO_CACHE_CREDENTIALS" ] || (echo "Uploading /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz to GCloud." && time gsutil cp /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/$SHALLOW_CLONE_TAR_FILENAME.gz); - fi - # Disable the full repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well) - - if [ -n "$DISABLE_SHALLOW_REPO_CACHING" ]; then - cd .. && rm -rf $CI_PROJECT_NAME; - echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME."; - time git clone --progress $CI_REPOSITORY_URL $CI_PROJECT_NAME; - cd $CI_PROJECT_NAME; - time git repack -d; - echo "Archiving $CI_PROJECT_NAME into /tmp/$FULL_CLONE_TAR_FILENAME."; - time git remote rm origin; - time tar cf /tmp/$FULL_CLONE_TAR_FILENAME .; - echo "GZipping /tmp/$FULL_CLONE_TAR_FILENAME."; - time gzip /tmp/$FULL_CLONE_TAR_FILENAME; - [ -z "$CI_REPO_CACHE_CREDENTIALS" ] || (echo "Uploading /tmp/$FULL_CLONE_TAR_FILENAME.gz to GCloud." && time gsutil cp /tmp/$FULL_CLONE_TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/$FULL_CLONE_TAR_FILENAME.gz); - fi diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index f4d8698f22d..6aa3d533315 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -44,8 +44,8 @@ docs-lint markdown: - .default-retry - .docs:rules:docs-lint # When updating the image version here, update it in /scripts/lint-doc.sh too. - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.10.4-markdownlint-0.28.1 - stage: test + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.12.0-markdownlint-0.29.0 + stage: lint needs: [] script: - scripts/lint-doc.sh @@ -53,8 +53,8 @@ docs-lint markdown: docs-lint links: extends: - .docs:rules:docs-lint - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-db71f027 - stage: test + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-0fae0f62 + stage: lint needs: [] script: # Prepare docs for build @@ -71,7 +71,21 @@ ui-docs-links lint: extends: - .docs:rules:docs-lint - .static-analysis-base - stage: test + - .ruby-cache + stage: lint needs: [] script: - bundle exec haml-lint -i DocumentationLinks + +docs-lint deprecations: + variables: + SETUP_DB: "false" + extends: + - .default-retry + - .rails-cache + - .default-before_script + - .docs:rules:deprecations + stage: lint + needs: [] + script: + - bundle exec rake gitlab:docs:check_deprecations diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 6974d63a49c..d3844d01213 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -11,7 +11,7 @@ - .default-retry - .default-before_script - .assets-compile-cache - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 variables: SETUP_DB: "false" WEBPACK_VENDOR_DLL: "true" @@ -133,6 +133,7 @@ update-storybook-yarn-cache: paths: - tmp/tests/frontend/ - knapsack/ + - crystalball/ # Builds FOSS, and EE fixtures in the EE project. # Builds FOSS fixtures in the FOSS project. @@ -193,21 +194,10 @@ graphql-schema-dump: # Disable warnings in browserslist which can break on backports # https://github.com/browserslist/browserslist/blob/a287ec6/node.js#L367-L384 BROWSERSLIST_IGNORE_OLD_DATA: "true" - SETUP_DB: "false" before_script: - - !reference [.default-before_script, before_script] - *yarn-install stage: test -eslint-as-if-foss: - extends: - - .frontend-test-base - - .frontend:rules:eslint-as-if-foss - - .as-if-foss - needs: [] - script: - - run_timed_command "yarn run lint:eslint:all" - .jest-base: extends: .frontend-test-base script: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index d0c26d60066..22bd3b1e2e9 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -44,6 +44,7 @@ prefix: "gitaly-binaries" paths: - tmp/tests/gitaly/_build/bin/ + - tmp/tests/gitaly/_build/deps/git/install/ - tmp/tests/gitaly/config.toml - tmp/tests/gitaly/gitaly2.config.toml - tmp/tests/gitaly/internal/ @@ -139,6 +140,10 @@ - <<: *gitaly-binaries-cache policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. +.ruby-cache: + cache: + - *ruby-gems-cache + .rails-cache: cache: - *ruby-gems-cache @@ -150,7 +155,12 @@ - *node-modules-cache - *rubocop-cache -.static-analysis-cache-push: +.rubocop-job-cache: + cache: + - *ruby-gems-cache + - *rubocop-cache + +.rubocop-job-cache-push: cache: - *ruby-gems-cache # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache` - *rubocop-cache-push @@ -203,7 +213,7 @@ - *storybook-node-modules-cache-push .use-pg11: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -212,7 +222,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg12: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -221,7 +231,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg13: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -230,34 +240,34 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg11-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - - name: elasticsearch:7.11.1 + - name: elasticsearch:7.14.2 command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust .use-pg12-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - - name: elasticsearch:7.11.1 + - name: elasticsearch:7.14.2 command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust .use-pg13-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - - name: elasticsearch:7.11.1 + - name: elasticsearch:7.14.2 command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust diff --git a/.gitlab/ci/qa-report.gitlab-ci.yml b/.gitlab/ci/qa-report.gitlab-ci.yml new file mode 100644 index 00000000000..61cbcfd58da --- /dev/null +++ b/.gitlab/ci/qa-report.gitlab-ci.yml @@ -0,0 +1,15 @@ +test-reliability-report: + extends: + - .qa:rules:reliable-reports:schedule + image: + name: ${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_DEFAULT_BRANCH} + entrypoint: [""] + before_script: + - cd /home/gitlab/qa + script: + - echo "Generate report for 'staging-full' runs" + - bundle exec rake "reliable_spec_report[staging-full,30,true]" + - bundle exec rake "unreliable_spec_report[staging-full,30,true]" + - echo "Generate report for 'package-and-qa' runs" + - bundle exec rake "reliable_spec_report[package-and-qa,30,true]" + - bundle exec rake "unreliable_spec_report[package-and-qa,30,true]" diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 88e732c2e75..74fc5f2cdc0 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -55,14 +55,18 @@ update-qa-cache: before_script: - source scripts/utils.sh - install_gitlab_gem + - tooling/bin/find_change_diffs ${CHANGES_DIFFS_DIR} script: - - ./scripts/trigger-build omnibus - -package-and-qa: - extends: - - .package-and-qa-base - - .qa:rules:package-and-qa - # This job often times out, so temporarily use private runners and a long timeout: https://gitlab.com/gitlab-org/gitlab/-/issues/238563 + - | + tooling/bin/qa/package_and_qa_check ${CHANGES_DIFFS_DIR} && exit_code=$? + if [ $exit_code -eq 0 ]; then + ./scripts/trigger-build omnibus + elif [ $exit_code -eq 1 ]; then + exit 1 + else + echo "Downstream jobs will not be triggered because package_and_qa_check exited with code: $exit_code" + fi + # These jobs often time out, so temporarily use private runners and a long timeout: https://gitlab.com/gitlab-org/gitlab/-/issues/238563 tags: - prm timeout: 4h @@ -71,3 +75,39 @@ package-and-qa: artifacts: false - job: build-assets-image artifacts: false + - detect-tests + artifacts: + expire_in: 7d + paths: + - ${CHANGES_FILE} + - ${CHANGES_DIFFS_DIR}/* + variables: + CHANGES_FILE: tmp/changed_files.txt + CHANGES_DIFFS_DIR: tmp/diffs + +.package-and-qa-ff-base: + script: + - export GITLAB_QA_OPTIONS="--set-feature-flags $(scripts/changed-feature-flags --files $(cat $CHANGES_FILE | tr ' ' ',') --state $QA_FF_STATE)" + - echo $GITLAB_QA_OPTIONS + - ./scripts/trigger-build omnibus + +package-and-qa: + extends: + - .package-and-qa-base + - .qa:rules:package-and-qa + +package-and-qa-ff-enabled: + extends: + - .package-and-qa-base + - .package-and-qa-ff-base + - .qa:rules:package-and-qa:feature-flags + variables: + QA_FF_STATE: "enable" + +package-and-qa-ff-disabled: + extends: + - .package-and-qa-base + - .package-and-qa-ff-base + - .qa:rules:package-and-qa:feature-flags + variables: + QA_FF_STATE: "disable" diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 00f65ab7ca8..1fb1c887e56 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -6,11 +6,6 @@ - .default-before_script - .rails-cache -.minimal-bundle-install: - script: - - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:kerberos:metrics:omnibus:ed25519" - - bundle_install_script - .base-script: script: # Only install knapsack after bundle install! Otherwise oddly some native @@ -26,6 +21,8 @@ .decomposed-database-rspec: variables: DECOMPOSED_DB: "true" + GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: "main" + GITLAB_USE_MODEL_LOAD_BALANCING: "true" .rspec-base: extends: .rails-job-base @@ -77,6 +74,12 @@ - .use-pg12 needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] +.rspec-base-pg12-as-if-jh: + extends: + - .rspec-base + - .use-pg12 + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-jh", "detect-tests", "add-jh-folder"] + .rspec-base-pg13: extends: - .rspec-base @@ -92,6 +95,11 @@ - .rspec-base - .use-pg12-ee +.rspec-jh-base-pg12: + extends: + - .rspec-base-pg12-as-if-jh + - .use-pg12-ee + .rspec-ee-base-pg13: extends: - .rspec-base @@ -113,6 +121,12 @@ - .rspec-ee-base-geo - .use-pg12-ee +.rspec-jh-base-geo-pg12: + extends: + - .rspec-jh-base-pg12 + script: + - !reference [.rspec-ee-base-geo, script] + .rspec-ee-base-geo-pg13: extends: - .rspec-ee-base-geo @@ -165,10 +179,9 @@ setup-test-env: extends: - .rails-job-base - .setup-test-env-cache - - .rails:rules:code-backstage-qa + - .rails:rules:setup-test-env stage: prepare variables: - GITLAB_TEST_EAGER_LOAD: "0" SETUP_DB: "false" script: - run_timed_command "scripts/setup-test-env" @@ -181,6 +194,7 @@ setup-test-env: paths: - config/secrets.yml - tmp/tests/gitaly/_build/bin/ + - tmp/tests/gitaly/_build/deps/git/install - tmp/tests/gitaly/config.toml - tmp/tests/gitaly/gitaly2.config.toml - tmp/tests/gitaly/internal/ @@ -223,11 +237,11 @@ update-gitaly-binaries-cache: .coverage-base: extends: - .default-retry - - .default-before_script - .coverage-cache - variables: - SETUP_DB: "false" - USE_BUNDLE_INSTALL: "false" + before_script: + - source scripts/utils.sh + - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:kerberos:metrics:omnibus:ed25519" + - bundle_install_script rspec migration pg12: extends: @@ -324,26 +338,16 @@ rspec fast_spec_helper minimal: - .minimal-rspec-tests - .rails:rules:ee-and-foss-fast_spec_helper:minimal -db:migrate:reset: +db:rollback: extends: .db-job-base script: - - bundle exec rake db:migrate:reset - -db:check-schema: - extends: - - .db-job-base - - .rails:rules:ee-mr-and-default-branch-only - script: - - source scripts/schema_changed.sh - - scripts/validate_migration_timestamps + - bundle exec rake db:migrate VERSION=20181228175414 + - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true -db:check-migrations: - extends: - - .db-job-base - - .rails:rules:ee-and-foss-mr-with-migration +db:migrate:reset: + extends: .db-job-base script: - - scripts/validate_migration_schema - allow_failure: true + - bundle exec rake db:migrate:reset db:migrate-from-previous-major-version: extends: .db-job-base @@ -352,7 +356,8 @@ db:migrate-from-previous-major-version: SETUP_DB: "false" PROJECT_TO_CHECKOUT: "gitlab-foss" TAG_TO_CHECKOUT: "v13.12.9" - script: + before_script: + - !reference [.default-before_script, before_script] - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"' - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="${TAG_TO_CHECKOUT}-ee"' - retry 'git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT' @@ -361,13 +366,28 @@ db:migrate-from-previous-major-version: - run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu" - git checkout -f $CI_COMMIT_SHA - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh + script: - run_timed_command "bundle exec rake db:migrate" -db:rollback: - extends: .db-job-base +db:check-schema: + extends: + - db:migrate-from-previous-major-version + - .rails:rules:ee-mr-and-default-branch-only + variables: + TAG_TO_CHECKOUT: "v14.4.0" script: - - bundle exec rake db:migrate VERSION=20181228175414 - - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true + - run_timed_command "bundle exec rake db:migrate" + - scripts/schema_changed.sh + - scripts/validate_migration_timestamps + +db:check-migrations: + extends: + - .db-job-base + - .rails:rules:ee-and-foss-mr-with-migration + script: + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME --depth 20 + - scripts/validate_migration_schema + allow_failure: true db:gitlabcom-database-testing: extends: .rails:rules:db:gitlabcom-database-testing @@ -457,21 +477,38 @@ rspec:coverage: # so we use `dependencies` here. dependencies: - setup-test-env + # FOSS/EE jobs - rspec migration pg12 - rspec unit pg12 - rspec integration pg12 - rspec system pg12 + # FOSS/EE minimal jobs + - rspec migration pg12 minimal + - rspec unit pg12 minimal + - rspec integration pg12 minimal + - rspec system pg12 minimal + # EE jobs - rspec-ee migration pg12 - rspec-ee unit pg12 - rspec-ee integration pg12 - rspec-ee system pg12 + # EE minimal jobs + - rspec-ee migration pg12 minimal + - rspec-ee unit pg12 minimal + - rspec-ee integration pg12 minimal + - rspec-ee system pg12 minimal + # Geo jobs - rspec-ee unit pg12 geo - rspec-ee integration pg12 geo - rspec-ee system pg12 geo + # Geo minimal jobs + - rspec-ee unit pg12 geo minimal + - rspec-ee integration pg12 geo minimal + - rspec-ee system pg12 geo minimal + # Memory jobs - memory-static - memory-on-boot script: - - !reference [.minimal-bundle-install, script] - run_timed_command "bundle exec scripts/merge-simplecov" - run_timed_command "bundle exec scripts/gather-test-memory-data" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' @@ -490,15 +527,67 @@ rspec:feature-flags: - .coverage-base - .rails:rules:rspec-feature-flags stage: post-test - needs: ["static-analysis"] + needs: + - job: "feature-flags-usage" + - job: "haml-lint foss" + - job: "haml-lint ee" + optional: true script: - - !reference [.minimal-bundle-install, script] - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then run_timed_command "bundle exec scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1); else run_timed_command "bundle exec scripts/used-feature-flags"; fi +rspec:skipped-flaky-tests-report: + extends: + - .default-retry + - .rails:rules:skipped-flaky-tests-report + image: ruby:2.7-alpine + stage: post-test + # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) + # so we use `dependencies` here. + dependencies: + # FOSS/EE jobs + - rspec migration pg12 + - rspec unit pg12 + - rspec integration pg12 + - rspec system pg12 + # FOSS/EE minimal jobs + - rspec migration pg12 minimal + - rspec unit pg12 minimal + - rspec integration pg12 minimal + - rspec system pg12 minimal + # EE jobs + - rspec-ee migration pg12 + - rspec-ee unit pg12 + - rspec-ee integration pg12 + - rspec-ee system pg12 + # EE minimal jobs + - rspec-ee migration pg12 minimal + - rspec-ee unit pg12 minimal + - rspec-ee integration pg12 minimal + - rspec-ee system pg12 minimal + # Geo jobs + - rspec-ee unit pg12 geo + - rspec-ee integration pg12 geo + - rspec-ee system pg12 geo + # Geo minimal jobs + - rspec-ee unit pg12 geo minimal + - rspec-ee integration pg12 geo minimal + - rspec-ee system pg12 geo minimal + variables: + SKIPPED_FLAKY_TESTS_REPORT: skipped_flaky_tests_report.txt + before_script: + - 'echo "SKIP_FLAKY_TESTS_AUTOMATICALLY: $SKIP_FLAKY_TESTS_AUTOMATICALLY"' + - mkdir -p rspec_flaky + script: + - find rspec_flaky/ -type f -name 'skipped_flaky_tests_*_report.txt' -exec cat {} + >> "${SKIPPED_FLAKY_TESTS_REPORT}" + artifacts: + expire_in: 31d + paths: + - ${SKIPPED_FLAKY_TESTS_REPORT} + # EE/FOSS: default refs (MRs, default branch, schedules) jobs # ####################################################### @@ -577,6 +666,31 @@ rspec system pg12-as-if-foss decomposed: - .decomposed-database-rspec - .rails:rules:decomposed-databases +rspec migration pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rspec-base-migration + - .rails:rules:as-if-jh-rspec + - .rspec-migration-parallel + +rspec unit pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rails:rules:as-if-jh-rspec + - .rspec-unit-parallel + +rspec integration pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rails:rules:as-if-jh-rspec + - .rspec-integration-parallel + +rspec system pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rails:rules:as-if-jh-rspec + - .rspec-system-parallel + rspec-ee migration pg12: extends: - .rspec-ee-base-pg12 @@ -684,6 +798,83 @@ rspec-ee system pg12 geo minimal: - .minimal-rspec-tests - .rails:rules:ee-only-system:minimal +rspec-ee migration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rspec-base-migration + - .rails:rules:as-if-jh-rspec + - .rspec-ee-migration-parallel + +rspec-ee unit pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-unit-parallel + +rspec-ee integration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-integration-parallel + +rspec-ee system pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-system-parallel + +rspec-ee unit pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-unit-geo-parallel + +rspec-ee integration pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-ee system pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh migration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rspec-base-migration + - .rails:rules:as-if-jh-rspec + +rspec-jh unit pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh integration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh system pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh unit pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh integration pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh system pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + db:rollback geo: extends: - db:rollback @@ -884,5 +1075,24 @@ fail-pipeline-early: - install_gitlab_gem script: - fail_pipeline_early + +rspec rspec-pg12-rerun-previous-failed-tests: + extends: + - .rspec-base-pg12 + - .rails:rules:rerun-previous-failed-tests + stage: test + needs: ["setup-test-env", "compile-test-assets", "detect-previous-failed-tests"] + script: + - !reference [.base-script, script] + - rspec_rerun_previous_failed_tests tmp/previous_failed_tests/rspec_failed_files.txt + +rspec rspec-ee-pg12-rerun-previous-failed-tests: + extends: + - "rspec rspec-pg12-rerun-previous-failed-tests" + - .rspec-ee-base-pg12 + script: + - !reference [.base-script, script] + - rspec_rerun_previous_failed_tests tmp/previous_failed_tests/rspec_ee_failed_files.txt + # EE: Canonical MR pipelines ################################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index b581cf83d56..33efc3c4788 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -9,6 +9,7 @@ code_quality: extends: - .default-retry - .use-docker-in-docker + stage: lint artifacts: paths: - gl-code-quality-report.json # GitLab-specific @@ -20,6 +21,7 @@ code_quality: extends: - .default-retry - sast + stage: lint needs: [] artifacts: paths: @@ -54,6 +56,7 @@ gosec-sast: .secret-analyzer: extends: .default-retry + stage: lint needs: [] artifacts: paths: @@ -68,6 +71,7 @@ secret_detection: extends: - .default-retry - dependency_scanning + stage: lint needs: [] variables: DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp" # GitLab-specific @@ -92,6 +96,13 @@ retire-js-dependency_scanning: gemnasium-python-dependency_scanning: rules: !reference [".reports:rules:gemnasium-python-dependency_scanning", rules] +yarn-audit-dependency_scanning: + extends: .ds-analyzer + image: "registry.gitlab.com/gitlab-org/security-products/analyzers/npm-audit:1.4.0" + variables: + TOOL: yarn + rules: !reference [".reports:rules:yarn-audit-dependency_scanning", rules] + # Analyze dependencies for malicious behavior # See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter .package_hunter-base: @@ -134,6 +145,7 @@ package_hunter-bundler: license_scanning: extends: .default-retry + stage: lint needs: [] artifacts: expire_in: 1 week # GitLab-specific diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index 6fe9e39cb82..670175a6f11 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -35,10 +35,11 @@ review-build-cng: extends: - .default-retry image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3.5-kubectl1.17 + resource_group: "review/${CI_COMMIT_REF_NAME}" variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v5.2.1" + GITLAB_HELM_CHART_REF: "v5.4.1" environment: name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} @@ -51,7 +52,6 @@ review-deploy: - .review:rules:review-deploy stage: deploy needs: ["review-build-cng"] - resource_group: "review/${CI_COMMIT_REF_NAME}" before_script: - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) @@ -101,6 +101,7 @@ review-stop: extends: - .review-stop-base - .review:rules:review-stop - stage: post-qa + stage: deploy + needs: [] script: - delete_k8s_release_namespace diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml index 6b9d4feb3c8..07f8bd4e5fd 100644 --- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -14,6 +14,7 @@ GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_ADMIN_USERNAME: "root" GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITLAB_QA_ADMIN_ACCESS_TOKEN: "${REVIEW_APPS_ROOT_TOKEN}" GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" SIGNUP_DISABLED: "true" @@ -31,9 +32,23 @@ expire_in: 7 days when: always +.parallel-qa-base: + parallel: 5 + script: + - export KNAPSACK_REPORT_PATH=knapsack/master_report.json + - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb + - | + bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \ + -- \ + --color --format documentation \ + --format RspecJunitFormatter --out tmp/rspec.xml + artifacts: + reports: + junit: qa/tmp/rspec.xml + .allure-report-base: image: - name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.6 + name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.4.1 entrypoint: [""] stage: post-qa variables: @@ -64,24 +79,23 @@ review-qa-smoke: script: - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}" +review-qa-reliable: + extends: + - .review-qa-base + - .review:rules:review-qa-reliable + - .parallel-qa-base + variables: + QA_RUN_TYPE: review-qa-reliable + QA_SCENARIO: Test::Instance::Reliable + review-qa-all: extends: - .review-qa-base - .review:rules:review-qa-all + - .parallel-qa-base variables: QA_RUN_TYPE: review-qa-all - parallel: 5 - script: - - export KNAPSACK_REPORT_PATH=knapsack/master_report.json - - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb - - | - bin/test Test::Instance::All "${CI_ENVIRONMENT_URL}" \ - -- \ - --color --format documentation \ - --format RspecJunitFormatter --out tmp/rspec.xml - artifacts: - reports: - junit: qa/tmp/rspec.xml + QA_SCENARIO: Test::Instance::All review-performance: extends: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 8ddcf9c2094..cf53e810d01 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -58,6 +58,9 @@ .if-merge-request-labels-run-decomposed: &if-merge-request-labels-run-decomposed if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-decomposed/' +.if-merge-request-labels-run-review-app: &if-merge-request-labels-run-review-app + if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-review-app/' + .if-security-merge-request: &if-security-merge-request if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID' @@ -91,9 +94,6 @@ .if-dot-com-ee-nightly-schedule-child-pipeline: &if-dot-com-ee-nightly-schedule-child-pipeline if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY == "nightly"' -.if-cache-credentials-schedule: &if-cache-credentials-schedule - if: '$CI_REPO_CACHE_CREDENTIALS && $CI_PIPELINE_SOURCE == "schedule"' - .if-dot-com-gitlab-org-default-branch: &if-dot-com-gitlab-org-default-branch if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' @@ -103,6 +103,9 @@ .if-dot-com-gitlab-org-and-security-merge-request: &if-dot-com-gitlab-org-and-security-merge-request if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_MERGE_REQUEST_IID' +.if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-qa: &if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-qa + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_MERGE_REQUEST_IID && $QA_MANUAL_FF_PACKAGE_AND_QA' + .if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_COMMIT_TAG' @@ -112,6 +115,9 @@ .if-security-pipeline-merge-result: &if-security-pipeline-merge-result if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == "gitlab-org/security" && $GITLAB_USER_LOGIN == "gitlab-release-tools-bot"' +.if-skip-flaky-tests-automatically: &if-skip-flaky-tests-automatically + if: '$SKIP_FLAKY_TESTS_AUTOMATICALLY == "true"' + #################### # Changes patterns # #################### @@ -159,11 +165,19 @@ - ".markdownlint.yml" - "scripts/lint-doc.sh" +.docs-deprecations-patterns: &docs-deprecations-patterns + - "doc/deprecations/index.md" + - "data/deprecations/*.yml" + - "data/deprecations/templates/_deprecation_template.md.erb" + - "lib/tasks/gitlab/docs/compile_deprecations.rake" + - "tooling/deprecations/docs.rb" + .bundler-patterns: &bundler-patterns - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' .nodejs-patterns: &nodejs-patterns - '{package.json,*/package.json,*/*/package.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' .python-patterns: &python-patterns - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' @@ -211,6 +225,7 @@ .startup-css-patterns: &startup-css-patterns - "{,ee/,jh/}app/assets/stylesheets/startup/**/*" +# Backend patterns + .ci-patterns .backend-patterns: &backend-patterns - "{,jh/}Gemfile{,.lock}" - "Rakefile" @@ -224,6 +239,7 @@ - ".gitlab/ci/**/*" - "*_VERSION" +# DB patterns + .ci-patterns .db-patterns: &db-patterns - "{,ee/,jh/}{,spec/}{db,migrations}/**/*" - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/database/**/*" @@ -252,6 +268,11 @@ - "{,ee/,jh/}spec/**/*" - "{,spec/}tooling/**/*" +.qa-patterns: &qa-patterns + - ".dockerignore" + - "qa/**/*" + +# Code patterns + .ci-patterns + .workhorse-patterns .code-patterns: &code-patterns - "{package.json,yarn.lock}" - ".browserslistrc" @@ -260,9 +281,6 @@ - ".csscomb.json" - "Dockerfile.assets" - "vendor/assets/**/*" - # CI changes - - ".gitlab-ci.yml" - - ".gitlab/ci/**/*" - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,rubocop_manual_todo}.yml" - "*_VERSION" @@ -273,11 +291,11 @@ - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" + # CI changes + - ".gitlab-ci.yml" + - ".gitlab/ci/**/*" -.qa-patterns: &qa-patterns - - ".dockerignore" - - "qa/**/*" - +# .code-patterns + .backstage-patterns .code-backstage-patterns: &code-backstage-patterns - "{package.json,yarn.lock}" - ".browserslistrc" @@ -286,9 +304,6 @@ - ".csscomb.json" - "Dockerfile.assets" - "vendor/assets/**/*" - # CI changes - - ".gitlab-ci.yml" - - ".gitlab/ci/**/*" - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,rubocop_manual_todo}.yml" - "*_VERSION" @@ -299,6 +314,9 @@ - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" + # CI changes + - ".gitlab-ci.yml" + - ".gitlab/ci/**/*" # Backstage changes - "Dangerfile" - "danger/**/*" @@ -307,6 +325,7 @@ - "{,ee/,jh/}spec/**/*" - "{,spec/}tooling/**/*" +# .code-patterns + .qa-patterns .code-qa-patterns: &code-qa-patterns - "{package.json,yarn.lock}" - ".browserslistrc" @@ -315,9 +334,6 @@ - ".csscomb.json" - "Dockerfile.assets" - "vendor/assets/**/*" - # CI changes - - ".gitlab-ci.yml" - - ".gitlab/ci/**/*" - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,rubocop_manual_todo}.yml" - "*_VERSION" @@ -328,10 +344,14 @@ - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" + # CI changes + - ".gitlab-ci.yml" + - ".gitlab/ci/**/*" # QA changes - ".dockerignore" - "qa/**/*" +# .code-patterns + .backstage-patterns + .qa-patterns .code-backstage-qa-patterns: &code-backstage-qa-patterns - "{package.json,yarn.lock}" - ".browserslistrc" @@ -340,9 +360,39 @@ - ".csscomb.json" - "Dockerfile.assets" - "vendor/assets/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,rubocop_manual_todo}.yml" + - "*_VERSION" + - "{,jh/}Gemfile{,.lock}" + - "Rakefile" + - "tests.yml" + - "config.ru" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "doc/api/graphql/reference/*" # Files in this folder are auto-generated + - "data/whats_new/*.yml" # CI changes - ".gitlab-ci.yml" - ".gitlab/ci/**/*" + # Backstage changes + - "Dangerfile" + - "danger/**/*" + - "{,ee/,jh/}fixtures/**/*" + - "{,ee/,jh/}rubocop/**/*" + - "{,ee/,jh/}spec/**/*" + - "{,spec/}tooling/**/*" + # QA changes + - ".dockerignore" + - "qa/**/*" + +# .code-backstage-qa-patterns + .workhorse-patterns +.setup-test-env-patterns: &setup-test-env-patterns + - "{package.json,yarn.lock}" + - ".browserslistrc" + - "babel.config.js" + - "jest.config.{base,integration,unit}.js" + - ".csscomb.json" + - "Dockerfile.assets" + - "vendor/assets/**/*" - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,rubocop_manual_todo}.yml" - "*_VERSION" @@ -353,6 +403,9 @@ - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" + # CI changes + - ".gitlab-ci.yml" + - ".gitlab/ci/**/*" # Backstage changes - "Dangerfile" - "danger/**/*" @@ -363,9 +416,12 @@ # QA changes - ".dockerignore" - "qa/**/*" + # Workhorse changes + - "GITLAB_WORKHORSE_VERSION" + - "workhorse/**/*" + - ".gitlab/ci/workhorse.gitlab-ci.yml" -.code-backstage-danger-patterns: &code-backstage-danger-patterns - # Backstage changes +.danger-patterns: &danger-patterns - "Dangerfile" - "danger/**/*" - "tooling/danger/**/*" @@ -381,6 +437,9 @@ - "config/helpers/**/*.js" - "vendor/assets/javascripts/**/*" +.feature-flag-development-config-patterns: &feature-flag-development-config-patterns + - "{,ee/}config/feature_flags/{development,ops}/*.yml" + ################ # Shared rules # ################ @@ -402,6 +461,7 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-and-security-merge-request changes: *ci-build-images-patterns - <<: *if-dot-com-gitlab-org-and-security-merge-request @@ -414,18 +474,11 @@ rules: - <<: *if-not-canonical-namespace when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-auto-deploy-branches - changes: *ci-build-images-patterns - changes: *code-qa-patterns -#################### -# Cache repo rules # -#################### -.cache-repo:rules: - rules: - - <<: *if-cache-credentials-schedule - allow_failure: true - ############# # CNG rules # ############# @@ -476,6 +529,11 @@ changes: *docs-patterns when: on_success +.docs:rules:deprecations: + rules: + - <<: *if-default-refs + changes: *docs-deprecations-patterns + ################## # GraphQL rules # ################## @@ -495,6 +553,7 @@ rules: - <<: *if-not-canonical-namespace when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-auto-deploy-branches - changes: *code-qa-patterns @@ -525,8 +584,8 @@ .frontend:rules:default-frontend-jobs: rules: - - <<: *if-default-refs - changes: *code-backstage-patterns + - <<: *if-merge-request-labels-run-all-rspec + - changes: *code-backstage-patterns .frontend:rules:default-frontend-jobs-as-if-foss: rules: @@ -683,6 +742,9 @@ - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-and-security-merge-request + changes: *feature-flag-development-config-patterns + when: never + - <<: *if-dot-com-gitlab-org-and-security-merge-request changes: *ci-qa-patterns allow_failure: true - <<: *if-dot-com-gitlab-org-and-security-merge-request @@ -695,12 +757,35 @@ - <<: *if-dot-com-gitlab-org-schedule allow_failure: true +.qa:rules:package-and-qa:feature-flags: + rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-qa + changes: *feature-flag-development-config-patterns + when: manual + allow_failure: true + - <<: *if-dot-com-gitlab-org-and-security-merge-request + changes: *feature-flag-development-config-patterns + allow_failure: true + +.qa:rules:reliable-reports:schedule: + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $QA_RELIABLE_REPORT == "true"' + allow_failure: true + ############### # Rails rules # ############### +.rails:rules:setup-test-env: + rules: + - changes: *setup-test-env-patterns + - <<: *if-merge-request-labels-run-all-rspec + .rails:rules:decomposed-databases: rules: - <<: *if-merge-request-labels-run-decomposed + - <<: *if-default-branch-schedule-nightly .rails:rules:ee-and-foss-migration: rules: @@ -1163,6 +1248,21 @@ - <<: *if-merge-request-labels-as-if-foss changes: *code-backstage-patterns +.rails:rules:as-if-jh-rspec: + rules: + - <<: *if-not-ee + when: never + - <<: *if-jh + when: never + - <<: *if-security-merge-request + changes: *code-backstage-patterns + allow_failure: true + - <<: *if-merge-request-labels-as-if-jh + allow_failure: true + - <<: *if-merge-request + changes: *ci-patterns + allow_failure: true + .rails:rules:ee-and-foss-db-library-code: rules: - changes: *db-library-patterns @@ -1180,9 +1280,25 @@ .rails:rules:detect-tests: rules: - - changes: *code-backstage-patterns + - changes: *code-backstage-qa-patterns - <<: *if-merge-request-labels-run-all-rspec +.rails:rules:detect-previous-failed-tests: + rules: + - <<: *if-security-merge-request + when: never + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *code-backstage-patterns + +.rails:rules:rerun-previous-failed-tests: + rules: + - <<: *if-security-merge-request + when: never + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *code-backstage-patterns + .rails:rules:rspec-foss-impact: rules: - <<: *if-not-ee @@ -1252,6 +1368,14 @@ when: never - changes: *code-backstage-patterns +.rails:rules:skipped-flaky-tests-report: + rules: + - <<: *if-not-ee + when: never + - <<: *if-skip-flaky-tests-automatically + changes: *code-backstage-patterns + - changes: *ci-patterns + ######################### # Static analysis rules # ######################### @@ -1260,6 +1384,12 @@ rules: - changes: *code-backstage-qa-patterns +.static-analysis:rules:ee: + rules: + - <<: *if-not-ee + when: never + - changes: *code-backstage-qa-patterns + .static-analysis:rules:as-if-foss: rules: - <<: *if-not-ee @@ -1370,6 +1500,12 @@ when: never - changes: *python-patterns +.reports:rules:yarn-audit-dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/' + when: never + - changes: *nodejs-patterns + .reports:rules:schedule-dast: rules: - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' @@ -1405,6 +1541,7 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request @@ -1415,13 +1552,13 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns - allow_failure: true - <<: *if-dot-com-gitlab-org-schedule .review:rules:review-build-cng: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request @@ -1431,24 +1568,22 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns - allow_failure: true - <<: *if-dot-com-gitlab-org-schedule-child-pipeline .review:rules:review-deploy: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns - allow_failure: true - <<: *if-dot-com-gitlab-org-schedule-child-pipeline allow_failure: true @@ -1458,6 +1593,7 @@ when: never - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request @@ -1473,6 +1609,7 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns @@ -1480,24 +1617,32 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request - changes: *code-qa-patterns + changes: *qa-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-patterns allow_failure: true - <<: *if-dot-com-ee-schedule-child-pipeline allow_failure: true # The rule needs to be duplicated between `on_success` and `on_failure` # because the jobs `needs` the previous job to complete. +# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs +# would start running before the qa jobs have started. # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 .review:rules:review-qa-smoke-report: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app + when: on_success + - <<: *if-merge-request-labels-run-review-app + when: on_failure - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns when: on_success @@ -1521,10 +1666,16 @@ - <<: *if-dot-com-ee-schedule-child-pipeline when: on_failure +.review:rules:review-qa-reliable: + rules: + - when: on_success + allow_failure: true + .review:rules:review-qa-all: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual @@ -1537,11 +1688,17 @@ # The rule needs to be duplicated between `on_success` and `on_failure` # because the jobs `needs` the previous job to complete. +# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs +# would start running before the qa jobs have started. # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 .review:rules:review-qa-all-report: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app + when: on_success + - <<: *if-merge-request-labels-run-review-app + when: on_failure - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual @@ -1569,13 +1726,16 @@ changes: *code-qa-patterns when: manual allow_failure: true - - <<: *if-dot-com-gitlab-org-schedule-child-pipeline + - <<: *if-dot-com-gitlab-org-schedule allow_failure: true .review:rules:review-stop: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-run-review-app + when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: manual @@ -1588,7 +1748,7 @@ .review:rules:danger-local: rules: - if: '$CI_MERGE_REQUEST_IID' - changes: *code-backstage-danger-patterns + changes: *danger-patterns ############### # Setup rules # @@ -1616,7 +1776,7 @@ changes: *code-backstage-patterns when: on_success -.setup:rules:no_ee_check: +.setup:rules:no-ee-check: rules: - <<: *if-not-foss when: never @@ -1624,6 +1784,14 @@ changes: *code-backstage-patterns when: on_success +.setup:rules:no-jh-check: + rules: + - <<: *if-jh + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + .setup:rules:verify-tests-yml: rules: - <<: *if-not-ee diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index eb7a5afad3d..4c674f38939 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -41,13 +41,21 @@ gitlab_git_test: script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes -no_ee_check: +no-ee-check: extends: - .minimal-job - - .setup:rules:no_ee_check + - .setup:rules:no-ee-check stage: test script: - - scripts/no-ee-check + - scripts/no-dir-check ee + +no-jh-check: + extends: + - .minimal-job + - .setup:rules:no-jh-check + stage: test + script: + - scripts/no-dir-check jh verify-tests-yml: extends: @@ -102,6 +110,23 @@ detect-tests as-if-foss: before_script: - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' +detect-previous-failed-tests: + extends: + - .detect-test-base + - .rails:rules:detect-previous-failed-tests + variables: + PREVIOUS_FAILED_TESTS_DIR: tmp/previous_failed_tests/ + RSPEC_PG_REGEX: /rspec .+ pg12( .+)?/ + RSPEC_EE_PG_REGEX: /rspec-ee .+ pg12( .+)?/ + script: + - source ./scripts/utils.sh + - source ./scripts/rspec_helpers.sh + - retrieve_previous_failed_tests ${PREVIOUS_FAILED_TESTS_DIR} "${RSPEC_PG_REGEX}" "${RSPEC_EE_PG_REGEX}" + artifacts: + expire_in: 7d + paths: + - ${PREVIOUS_FAILED_TESTS_DIR} + add-jh-folder: extends: .setup:rules:add-jh-folder image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge @@ -112,6 +137,7 @@ add-jh-folder: - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/main-jh/gitlab-main-jh.tar.gz?path=jh" - tar -xf "jh-folder.tar.gz" - mv gitlab-main-jh-jh/jh/ ./ + - cp Gemfile.lock jh/ - ls -l jh/ artifacts: expire_in: 2d diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml index 85df68e9030..ebd223c4171 100644 --- a/.gitlab/ci/static-analysis.gitlab-ci.yml +++ b/.gitlab/ci/static-analysis.gitlab-ci.yml @@ -2,7 +2,7 @@ extends: - .default-retry - .default-before_script - - .static-analysis-cache + stage: lint needs: [] variables: SETUP_DB: "false" @@ -14,7 +14,7 @@ update-static-analysis-cache: extends: - .static-analysis-base - - .static-analysis-cache-push + - .rubocop-job-cache-push - .shared:rules:update-cache stage: prepare script: @@ -23,31 +23,95 @@ update-static-analysis-cache: static-analysis: extends: - .static-analysis-base + - .static-analysis-cache - .static-analysis:rules:ee-and-foss - stage: test - parallel: 4 + parallel: 2 script: - run_timed_command "retry yarn install --frozen-lockfile" - scripts/static-analysis - artifacts: - expire_in: 31d - when: always - paths: - - tmp/feature_flags/ -static-analysis-with-database: +static-analysis as-if-foss: + extends: + - static-analysis + - .static-analysis:rules:as-if-foss + - .as-if-foss + +static-verification-with-database: extends: - .static-analysis-base + - .rubocop-job-cache - .static-analysis:rules:ee-and-foss - .use-pg12 - stage: test script: - bundle exec rake lint:static_verification_with_database variables: SETUP_DB: "true" -static-analysis as-if-foss: +eslint: extends: - - static-analysis - - .static-analysis:rules:as-if-foss + - .static-analysis-base + - .yarn-cache + - .static-analysis:rules:ee + needs: [] + variables: + USE_BUNDLE_INSTALL: "false" + script: + - run_timed_command "retry yarn install --frozen-lockfile" + - run_timed_command "yarn run lint:eslint:all" + +eslint as-if-foss: + extends: + - eslint + - .frontend:rules:eslint-as-if-foss - .as-if-foss + +haml-lint foss: + extends: + - .static-analysis-base + - .ruby-cache + - .static-analysis:rules:ee-and-foss + script: + - run_timed_command "bin/rake 'haml_lint[app/views]'" + artifacts: + expire_in: 31d + when: always + paths: + - tmp/feature_flags/ + +haml-lint ee: + extends: + - "haml-lint foss" + - .static-analysis:rules:ee + script: + - run_timed_command "bin/rake 'haml_lint[ee/app/views]'" + +rubocop: + extends: + - .static-analysis-base + - .rubocop-job-cache + - .static-analysis:rules:ee-and-foss + script: + - run_timed_command "bundle exec rubocop --parallel" + +qa:testcases: + extends: + - .static-analysis-base + - .rubocop-job-cache + - .static-analysis:rules:ee-and-foss + script: + - run_timed_command "bundle exec rubocop qa/qa/specs/features/**/* --only QA/DuplicateTestcaseLink" + +feature-flags-usage: + extends: + - .static-analysis-base + - .rubocop-job-cache + - .static-analysis:rules:ee-and-foss + script: + # We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used, + # the cache would prevent these files from being created. + - run_timed_command "bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false" + artifacts: + expire_in: 31d + when: always + paths: + - tmp/feature_flags/ diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index 0da0a334699..47b5d29ab8e 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -9,24 +9,21 @@ workhorse:verify: .workhorse:test: extends: .workhorse:rules:workhorse - services: - - name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest - # Disable the hooks so we don't have to stub the GitLab API - command: ["/usr/bin/env", "GITALY_TESTING_NO_GIT_HOOKS=1", "/scripts/process-wrapper"] - alias: gitaly variables: - GITALY_ADDRESS: "tcp://gitaly:8075" + GITALY_ADDRESS: "tcp://127.0.0.1:8075" stage: test - needs: [] + needs: + - setup-test-env script: - go version - apt-get update && apt-get -y install libimage-exiftool-perl + - scripts/gitaly-test-build - make -C workhorse test workhorse:test using go 1.16: extends: .workhorse:test - image: ${GITLAB_DEPENDENCY_PROXY}golang:1.16 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-golang-1.16-git-2.31 workhorse:test using go 1.17: extends: .workhorse:test - image: ${GITLAB_DEPENDENCY_PROXY}golang:1.17 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-golang-1.17-git-2.31 diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index b25ad55e0ce..590593b9d75 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -5,7 +5,7 @@ lint-yaml: - .default-retry - .yaml-lint:rules image: pipelinecomponents/yamllint:latest - stage: test + stage: lint needs: [] variables: LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates |