diff options
Diffstat (limited to '.gitlab')
| -rw-r--r-- | .gitlab/CODEOWNERS | 11 | ||||
| -rw-r--r-- | .gitlab/ci/dast.gitlab-ci.yml | 203 | ||||
| -rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | .gitlab/ci/notify.gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 54 | ||||
| -rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 50 | ||||
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 17 | ||||
| -rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 100 | ||||
| -rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 43 | ||||
| -rw-r--r-- | .gitlab/issue_templates/Bug.md | 26 | ||||
| -rw-r--r-- | .gitlab/issue_templates/Feature proposal.md | 6 | ||||
| -rw-r--r-- | .gitlab/issue_templates/QA Failure.md | 8 | ||||
| -rw-r--r-- | .gitlab/issue_templates/Security developer workflow.md | 2 | ||||
| -rw-r--r-- | .gitlab/issue_templates/actionable_insight.md | 4 | ||||
| -rw-r--r-- | .gitlab/merge_request_templates/Documentation.md | 13 | ||||
| -rw-r--r-- | .gitlab/merge_request_templates/Security Release.md | 1 |
16 files changed, 454 insertions, 88 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 7a5516338e8..b4fd436cc58 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -10,13 +10,13 @@ /doc/ @gl-docsteam /doc/administration/monitoring/ @aqualls /doc/development/ @marcia @mjang1 -/doc/development/documentation/ @mikelewis +/doc/development/documentation/ @cnorris /doc/ci @marcel.amirault @sselhorn /doc/operations @aqualls @eread /doc/user/clusters @aqualls /doc/user/infrastructure @aqualls /doc/user/project/clusters @aqualls -/doc/.vale/ @marcel.amirault @eread @aqualls @mikelewis +/doc/.vale/ @marcel.amirault @eread @aqualls @cnorris [Docs Create] /doc/user/project/merge_requests/allow_collaboration.md @marcia @@ -165,12 +165,15 @@ /.gitlab/ci/ @gl-quality/eng-prod /.gitlab/ci/docs.gitlab-ci.yml @gl-quality/eng-prod @gl-docsteam /.gitlab/ci/releases.gitlab-ci.yml @gl-quality/eng-prod @gitlab-org/delivery +/.gitlab/ci/dast.gitlab-ci.yml @dappelt @ngeorge1 @gl-quality/eng-prod +/.gitlab/ci/reports.gitlab-ci.yml @gitlab-com/gl-security/appsec @gl-quality/eng-prod /.gitlab/CODEOWNERS @gl-quality/eng-prod Dangerfile @gl-quality/eng-prod /danger/ @gl-quality/eng-prod /lib/gitlab/danger/ @gl-quality/eng-prod /scripts/ @gl-quality/eng-prod /scripts/frontend/ @gl-quality/eng-prod @gitlab-org/maintainers/frontend +/scripts/review_apps/seed-dast-test-data.sh @dappelt @ngeorge1 @gl-quality/eng-prod .editorconfig @gl-quality/eng-prod [End-to-end] @@ -192,9 +195,11 @@ Dangerfile @gl-quality/eng-prod # Secure & Threat Management ownership delineation # https://about.gitlab.com/handbook/engineering/development/threat-management/delineate-secure-threat-management.html#technical-boundaries [Secure] -/ee/app/models/vulnerability.rb @gitlab-org/secure/threat-insights-backend-team +/ee/app/finders/security/ @gitlab-org/secure/threat-insights-backend-team /ee/app/models/security/ @gitlab-org/secure/threat-insights-backend-team /ee/app/models/vulnerabilities/ @gitlab-org/secure/threat-insights-backend-team +/ee/app/models/vulnerability.rb @gitlab-org/secure/threat-insights-backend-team +/ee/lib/api/vulnerabilit*.rb @gitlab-org/secure/threat-insights-backend-team /ee/lib/gitlab/ci/parsers/license_compliance/ @gitlab-org/secure/composition-analysis-be /ee/lib/gitlab/ci/parsers/security/ @gitlab-org/secure/composition-analysis-be @gitlab-org/secure/dynamic-analysis-be @gitlab-org/secure/static-analysis-be @gitlab-org/secure/fuzzing-be /ee/lib/gitlab/ci/reports/coverage_fuzzing/ @gitlab-org/secure/fuzzing-be diff --git a/.gitlab/ci/dast.gitlab-ci.yml b/.gitlab/ci/dast.gitlab-ci.yml new file mode 100644 index 00000000000..93f64930822 --- /dev/null +++ b/.gitlab/ci/dast.gitlab-ci.yml @@ -0,0 +1,203 @@ +.dast_conf: + tags: + - prm + # For scheduling dast job + extends: + - .reports:schedule-dast + image: + name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" + resource_group: dast_scan + variables: + DAST_USERNAME_FIELD: "user[login]" + DAST_PASSWORD_FIELD: "user[password]" + DAST_FULL_SCAN_ENABLED: "true" + DAST_SPIDER_MINS: 0 + # TBD pin to a version + DAST_VERSION: 1.22.1 + # -Xmx is used to set the JVM memory to 6GB to prevent DAST OutOfMemoryError. + DAST_ZAP_CLI_OPTIONS: "-Xmx6144m" + DAST_RULES: "41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,90019,10109,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028,10003,50003,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90020,90022,90033" + before_script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # Below three lines can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/230687 is fixed + - mkdir -p /zap/xml + - 'sed -i "84 s/true/false/" /zap/xml/config.xml' + - cat /zap/xml/config.xml + # Help pages are excluded from scan as they are static pages. + # profile/two_factor_auth is excluded from scan to prevent 2FA from being turned on from user profile, which will reduce coverage. + - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/help/.*,${DAST_WEBSITE}/profile/two_factor_auth,${DAST_WEBSITE}/users/sign_out"' + - enable_rule () { read all_rules; rule=$1; echo $all_rules | sed -r "s/(,)?$rule(,)?/\1-1\2/" ; } + # Sort ids in DAST_RULES ascendingly, which is required when using DAST_RULES as argument to enable_rule + - 'DAST_RULES=$(echo $DAST_RULES | tr "," "\n" | sort -n | paste -sd ",")' + needs: ["review-deploy"] + stage: dast + # Default job timeout set to 90m and dast rules needs 2h to so that it won't timeout. + timeout: 2h + # Add retry because of intermittent connection problems. See https://gitlab.com/gitlab-org/gitlab/-/issues/244313 + retry: 1 + artifacts: + paths: + - gl-dast-report.json # GitLab-specific + reports: + dast: gl-dast-report.json + expire_in: 1 week # GitLab-specific + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset1: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user1" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10019 | enable_rule 10020 | enable_rule 10021 | enable_rule 10023 | enable_rule 10024 | enable_rule 10025 | enable_rule 10037 | enable_rule 10040 | enable_rule 10054 | enable_rule 10055 | enable_rule 10056) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset2: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user2" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90011 | enable_rule 90020 | enable_rule 90022 | enable_rule 90033) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset3: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user3" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40016 | enable_rule 40017 | enable_rule 50000 | enable_rule 50001) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset4: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user4" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 0 | enable_rule 2 | enable_rule 3 | enable_rule 7 ) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset5: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user5" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10010 | enable_rule 10011 | enable_rule 10015 | enable_rule 10017 | enable_rule 10019) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset6: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user6" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 30001 | enable_rule 40009) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed +# DAST scan with a subset of Beta scan rules. +# DAST-fullscan-ruleset7: +# extends: +# - .dast_conf +# variables: +# DAST_USERNAME: "user7" +# script: +# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10098 | enable_rule 10105 | enable_rule 10202 | enable_rule 30002 | enable_rule 40003 | enable_rule 40008 | enable_rule 40009) +# - echo $DAST_EXCLUDE_RULES +# - /analyze -t $DAST_WEBSITE -d + +# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed +# Below jobs runs DAST scans with one time consuming scan rule. These scan rules are disabled in above jobs so that those jobs won't timeout. +# DAST scan with rule - 20019 External Redirect +# DAST-fullscan-rule-20019: +# extends: +# - .dast_conf +# variables: +# DAST_USERNAME: "user8" +# script: +# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 20019) +# - echo $DAST_EXCLUDE_RULES +# - /analyze -t $DAST_WEBSITE -d + +# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed +# DAST scan with rule - 10107 Httpoxy - Proxy Header Misuse - Active/beta +# DAST-fullscan-rule-10107: +# extends: +# - .dast_conf +# variables: +# DAST_USERNAME: "user9" +# script: +# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10107) +# - echo $DAST_EXCLUDE_RULES +# - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 90020 Remote OS Command Injection +DAST-fullscan-rule-90020: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user10" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90020) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 40018 SQL Injection - Active/release +DAST-fullscan-rule-40018: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user11" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40018) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 40014 Cross Site Scripting (Persistent) - Active/release +DAST-fullscan-rule-40014: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user12" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40014) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 6 Path travesal +DAST-fullscan-rule-6: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user13" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 6) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 40012 Cross Site Scripting (Reflected) +DAST-fullscan-rule-40012: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user14" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40012) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 62546e59368..0e0e156a64f 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -42,7 +42,7 @@ docs lint: extends: - .default-retry - .docs:rules:docs-lint - image: "registry.gitlab.com/gitlab-org/gitlab-docs:lint" + image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint:vale-2.3.4-markdownlint-0.23.2" stage: test needs: [] script: diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml index fcdd5ee97d2..6dcf19da942 100644 --- a/.gitlab/ci/notify.gitlab-ci.yml +++ b/.gitlab/ci/notify.gitlab-ci.yml @@ -10,7 +10,7 @@ notify-update-gitaly: extends: - .notify-slack rules: - - if: '$CI_MERGE_REQUEST_IID && $CI_COMMIT_BRANCH == $GITALY_UPDATE_BRANCH' + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $GITALY_UPDATE_BRANCH' when: on_failure allow_failure: true variables: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 0b54626f690..165476678bb 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -296,6 +296,21 @@ gitlab:setup: paths: - log/*.log +db:backup_and_restore: + extends: .db-job-base + variables: + SETUP_DB: "false" + GITLAB_ASSUME_YES: "1" + script: + - . scripts/prepare_build.sh + - bundle exec rake db:drop db:create db:structure:load db:seed_fu + - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry} + - bundle exec rake gitlab:backup:create + - date + - bundle exec rake gitlab:backup:restore + rules: + - changes: ["lib/backup/**/*"] + rspec:coverage: extends: - .rails-job-base @@ -490,21 +505,50 @@ rspec-ee system pg12 geo: ################################################## # EE: Canonical MR pipelines +rspec fail-fast: + extends: + - .rspec-ee-base-pg11 # This job also runs EE spec which needs elasticsearch + - .rails:rules:rspec fail-fast + stage: test + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] + script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - source scripts/rspec_helpers.sh + - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine" + artifacts: + expire_in: 7d + paths: + - tmp/capybara/ + rspec foss-impact: extends: - .rspec-base-pg11-as-if-foss - - .rails:rules:ee-mr-only + - .rails:rules:rspec-foss-impact + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"] script: - - install_gitlab_gem - run_timed_command "scripts/gitaly-test-build" - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - - tooling/bin/find_foss_tests tmp/matching_foss_tests.txt - - rspec_matched_tests tmp/matching_foss_tests.txt "--tag ~quarantine" + - rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine" artifacts: expire_in: 7d paths: - - tmp/matching_foss_tests.txt - tmp/capybara/ + +fail-pipeline-early: + extends: + - .rails:rules:fail-pipeline-early + stage: post-test + needs: + - job: rspec fail-fast + artifacts: false + variables: + GIT_DEPTH: 1 + before_script: + - source scripts/utils.sh + - install_api_client_dependencies_with_apt + script: + - fail_pipeline_early # EE: Canonical MR pipelines ################################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index b9f81f2eb0f..0e2f12789db 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -81,7 +81,13 @@ nodejs-scan-sast: secrets-sast: extends: .sast image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:3" + artifacts: + paths: + - gl-secret-detection-report.json # GitLab-specific + reports: + sast: gl-secret-detection-report.json + expire_in: 1 week # GitLab-specific # We need to duplicate this job's definition because it seems it's impossible to # override an included `only.refs`. @@ -145,45 +151,3 @@ dependency_scanning: reports: dependency_scanning: gl-dependency-scanning-report.json expire_in: 1 week # GitLab-specific - -# Temporarily disabling review apps -## We need to duplicate this job's definition because it seems it's impossible to -## override an included `only.refs`. -## See https://gitlab.com/gitlab-org/gitlab/issues/31371. -# dast: -# extends: -# - .default-retry -# - .reports:rules:dast -# # This is needed so that manual jobs with needs don't block the pipeline. -# # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. -# dependencies: ["review-deploy"] -# stage: qa # GitLab-specific -# image: -# name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" -# variables: -# # To be done in a later iteration -# # DAST_USERNAME: "root" -# # DAST_USERNAME_FIELD: "user[login]" -# # DAST_PASSWORD_FIELD: "user[passowrd]" -# DAST_VERSION: 1 -# script: -# - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' -# # To be done in a later iteration -# # - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' -# # - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' -# - /analyze -t $DAST_WEBSITE -# timeout: 4h -# artifacts: -# paths: -# - gl-dast-report.json # GitLab-specific -# reports: -# dast: gl-dast-report.json -# expire_in: 1 week # GitLab-specific - -# To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 -# schedule:dast: -# extends: -# - dast -# - .reports:schedule-dast -# variables: -# DAST_FULL_SCAN_ENABLED: "true" diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 4e3a80372a6..d34687cfdad 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -43,9 +43,9 @@ review-build-cng: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" REVIEW_APPS_DOMAIN: "temp.gitlab-review.app" # FIXME: using temporary domain DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v4.1.3" + GITLAB_HELM_CHART_REF: "v4.3.0" environment: - name: review/${CI_COMMIT_REF_NAME} + name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop auto_stop_in: 48 hours @@ -53,7 +53,7 @@ review-build-cng: review-deploy: extends: - .review-workflow-base - - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + - .review:rules:review-deploy stage: review dependencies: [] resource_group: "review/${CI_COMMIT_REF_NAME}" @@ -77,6 +77,11 @@ review-deploy: # to have to manually start the jobs in sequence, so we do it for them. - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"' - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"' + after_script: + # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. + # Set DAST_RUN to true when jobs are manually scheduled. + - if [ "$DAST_RUN" == "true" ]; then source scripts/review_apps/seed-dast-test-data.sh; TRACE=1 trigger_proj_user_creation; fi + artifacts: paths: [environment_url.txt] expire_in: 2 days @@ -108,8 +113,8 @@ review-stop-failed-deployment: review-stop: extends: - .review-stop-base - - .review:rules:mr-only-manual - stage: review + - .review:rules:review-stop + stage: post-qa script: - delete_release @@ -167,7 +172,7 @@ review-qa-all: review-performance: extends: - .default-retry - - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + - .review:rules:review-performance image: name: sitespeedio/sitespeed.io:6.3.1 entrypoint: [""] diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 839a06862b2..a8e0e1ccaaa 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -67,6 +67,12 @@ .if-cache-credentials-schedule: &if-cache-credentials-schedule if: '$CI_REPO_CACHE_CREDENTIALS && $CI_PIPELINE_SOURCE == "schedule"' +.if-rspec-fail-fast-disabled: &if-rspec-fail-fast-disabled + if: '$RSPEC_FAIL_FAST_ENABLED != "true"' + +.if-rspec-fail-fast-skipped: &if-rspec-fail-fast-skipped + if: '$CI_MERGE_REQUEST_TITLE =~ /SKIP RSPEC FAIL-FAST/' + #################### # Changes patterns # #################### @@ -83,6 +89,7 @@ - ".gitlab/ci/frontend.gitlab-ci.yml" - ".gitlab/ci/build-images.gitlab-ci.yml" - ".gitlab/ci/review.gitlab-ci.yml" + - "scripts/trigger-build" .ci-qa-patterns: &ci-qa-patterns - ".gitlab-ci.yml" @@ -121,11 +128,13 @@ - "{,ee/}spec/**/*.rb" - ".gitlab-ci.yml" - ".gitlab/ci/**/*" + - "*_VERSION" .db-patterns: &db-patterns - "{,ee/}{,spec/}{db,migrations}/**/*" - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*" - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer + - "{,ee/}app/models/project_statistics.rb" # Used to calculate sizes in migration specs .backstage-patterns: &backstage-patterns - "Dangerfile" @@ -147,6 +156,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -168,6 +178,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -191,6 +202,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -211,6 +223,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -506,6 +519,7 @@ - <<: *if-security-merge-request changes: *db-patterns - <<: *if-merge-request-title-as-if-foss + changes: *db-patterns - <<: *if-merge-request changes: *ci-patterns @@ -516,6 +530,7 @@ - <<: *if-security-merge-request changes: *backend-patterns - <<: *if-merge-request-title-as-if-foss + changes: *backend-patterns - <<: *if-merge-request changes: *ci-patterns @@ -526,6 +541,7 @@ - <<: *if-security-merge-request changes: *backend-patterns - <<: *if-merge-request-title-as-if-foss + changes: *backend-patterns - <<: *if-merge-request changes: *ci-patterns @@ -536,6 +552,7 @@ - <<: *if-security-merge-request changes: *code-backstage-patterns - <<: *if-merge-request-title-as-if-foss + changes: *code-backstage-patterns - <<: *if-merge-request changes: *ci-patterns @@ -549,7 +566,16 @@ - <<: *if-master-refs changes: *code-backstage-patterns -.rails:rules:ee-mr-only: +.rails:rules:detect-tests: + rules: + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-backstage-patterns + +.rails:rules:rspec-foss-impact: rules: - <<: *if-not-ee when: never @@ -560,6 +586,34 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *code-backstage-patterns +.rails:rules:rspec fail-fast: + rules: + - <<: *if-rspec-fail-fast-disabled + when: never + - <<: *if-rspec-fail-fast-skipped + when: never + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-backstage-patterns + +.rails:rules:fail-pipeline-early: + rules: + - <<: *if-rspec-fail-fast-disabled + when: never + - <<: *if-rspec-fail-fast-skipped + when: never + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *code-backstage-patterns + when: on_failure + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-backstage-patterns + when: on_failure + .rails:rules:downtime_check: rules: - <<: *if-merge-request @@ -569,6 +623,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request + changes: *code-backstage-patterns - <<: *if-master-schedule-2-hourly - <<: *if-merge-request-title-run-all-rspec @@ -643,7 +699,8 @@ rules: - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' when: never - - <<: *if-dot-com-gitlab-org-schedule + - <<: *if-master-schedule-nightly + allow_failure: true ################ # Review rules # @@ -662,8 +719,26 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-schedule -.review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise: +.review:rules:review-deploy: + rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-review-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *frontend-patterns + allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule + allow_failure: true + +.review:rules:review-performance: rules: + - if: '$DAST_RUN == "true"' # Skip this job when DAST is run + when: never - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-merge-request @@ -719,6 +794,17 @@ - <<: *if-dot-com-gitlab-org-schedule allow_failure: true +.review:rules:review-stop: + rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule + allow_failure: true + .review:rules:danger: rules: - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID' @@ -757,6 +843,14 @@ changes: *code-backstage-patterns when: on_success +.setup:rules:verify-tests-yml: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + ####################### # Test metadata rules # ####################### diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 26c7a2194cc..cf42d2a8a5e 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -48,3 +48,46 @@ no_ee_check: stage: test script: - scripts/no-ee-check + +verify-tests-yml: + extends: + - .setup:rules:verify-tests-yml + image: ruby:2.6-alpine + stage: test + needs: [] + script: + - source scripts/utils.sh + - install_tff_gem + - scripts/verify-tff-mapping + +.detect-test-base: + image: ruby:2.6-alpine + needs: [] + stage: prepare + script: + - source scripts/utils.sh + - install_gitlab_gem + - install_tff_gem + - tooling/bin/find_foss_tests ${MATCHED_TESTS_FILE} + - 'echo "test files affected: $(cat $MATCHED_TESTS_FILE)"' + artifacts: + expire_in: 7d + paths: + - ${MATCHED_TESTS_FILE} + +detect-tests: + extends: + - .detect-test-base + - .rails:rules:detect-tests + variables: + MATCHED_TESTS_FILE: tmp/matching_tests.txt + +detect-tests as-if-foss: + extends: + - .detect-test-base + - .rails:rules:detect-tests + - .as-if-foss + variables: + MATCHED_TESTS_FILE: tmp/matching_foss_tests.txt + before_script: + - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md index 037f83c93d2..41b694fdf2c 100644 --- a/.gitlab/issue_templates/Bug.md +++ b/.gitlab/issue_templates/Bug.md @@ -12,37 +12,39 @@ and verify the issue you're about to submit isn't a duplicate. ### Summary -(Summarize the bug encountered concisely) +<!-- Summarize the bug encountered concisely. --> ### Steps to reproduce -(How one can reproduce the issue - this is very important) +<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. --> ### Example Project -(If possible, please create an example project here on GitLab.com that exhibits the problematic behavior, and link to it here in the bug report) - -(If you are using an older version of GitLab, this will also determine whether the bug is fixed in a more recent version) +<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic +behavior, and link to it here in the bug report. If you are using an older version of GitLab, this +will also determine whether the bug is fixed in a more recent version. --> ### What is the current *bug* behavior? -(What actually happens) +<!-- Describe what actually happens. --> ### What is the expected *correct* behavior? -(What you should see instead) +<!-- Describe what you should see instead. --> ### Relevant logs and/or screenshots -(Paste any relevant logs - please use code blocks (```) to format console output, -logs, and code as it's tough to read otherwise.) +<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code + as it's tough to read otherwise. --> ### Output of checks -(If you are reporting a bug on GitLab.com, write: This bug happens on GitLab.com) +<!-- If you are reporting a bug on GitLab.com, write: This bug happens on GitLab.com --> #### Results of GitLab environment info +<!-- Input any relevant GitLab environment information if needed. --> + <details> <summary>Expand for output related to GitLab environment info</summary> @@ -59,6 +61,8 @@ logs, and code as it's tough to read otherwise.) #### Results of GitLab application Check +<!-- Input any relevant GitLab application check information if needed. --> + <details> <summary>Expand for output related to the GitLab application check</summary> <pre> @@ -76,6 +80,6 @@ logs, and code as it's tough to read otherwise.) ### Possible fixes -(If you can, link to the line of code that might be responsible for the problem) +<!-- If you can, link to the line of code that might be responsible for the problem. --> /label ~bug diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index 4e894b8ce80..0f19b7c75f5 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -1,4 +1,8 @@ -<!-- The first four sections: "Problem to solve", "Intended users", "User experience goal", and "Proposal", are strongly recommended, while the rest of the sections can be filled out during the problem validation or breakdown phase. However, keep in mind that providing complete and relevant information early helps our product team validate the problem and start working on a solution. --> +<!-- The first section "Release notes" is required if you want to have your release post blog MR auto generated. Currently in BETA, details on the **release post item generator** can be found in the handbook: https://about.gitlab.com/handbook/marketing/blog/release-posts/#release-post-item-generator and this video: https://www.youtube.com/watch?v=rfn9ebgTwKg. The next four sections: "Problem to solve", "Intended users", "User experience goal", and "Proposal", are strongly recommended in your first draft, while the rest of the sections can be filled out during the problem validation or breakdown phase. However, keep in mind that providing complete and relevant information early helps our product team validate the problem and start working on a solution. --> + +### Release notes + +<!-- What is the problem and solution you're proposing? This content sets the overall vision for the feature and serves as the release notes that will populate in various places, including the [release post blog](https://about.gitlab.com/releases/categories/releases/) and [Gitlab project releases](https://gitlab.com/gitlab-org/gitlab/-/releases). " --> ### Problem to solve diff --git a/.gitlab/issue_templates/QA Failure.md b/.gitlab/issue_templates/QA Failure.md index 772f363ae31..9751b40cb91 100644 --- a/.gitlab/issue_templates/QA Failure.md +++ b/.gitlab/issue_templates/QA Failure.md @@ -68,10 +68,10 @@ a nightly pipeline, select ~"found:nightly". <!-- https://about.gitlab.com/handbook/engineering/quality/guidelines/#priorities: -- ~P::1: Tests that are needed to verify fundamental GitLab functionality. -- ~P::2: Tests that deal with external integrations which may take a longer time to debug and fix. +- ~"priority::1": Tests that are needed to verify fundamental GitLab functionality. +- ~"priority::2": Tests that deal with external integrations which may take a longer time to debug and fix. --> -/label ~P:: +/label ~priority:: -<!-- Select the current milestone if ~P::1 or the next milestone if ~P::2. --> +<!-- Select the current milestone if ~"priority::1" or the next milestone if ~"priority::2". --> /milestone % diff --git a/.gitlab/issue_templates/Security developer workflow.md b/.gitlab/issue_templates/Security developer workflow.md index d21da6a161b..840ef4c6337 100644 --- a/.gitlab/issue_templates/Security developer workflow.md +++ b/.gitlab/issue_templates/Security developer workflow.md @@ -27,7 +27,7 @@ After your merge request has been approved according to our [approval guidelines * At this point, it might be easy to squash the commits from the MR into one * You can use the script `bin/secpick` instead of the following steps, to help you cherry-picking. See the [secpick documentation] - [ ] Create each MR targeting the stable branch `X-Y-stable`, using the [Security Release merge request template]. - * Every merge request will have its own set of TODOs, so make sure to complete those. + * Every merge request will have its own set of to-dos, so make sure to complete those. - [ ] On the "Related merge requests" section, ensure that `4` merge requests are associated: The one targeting `master` and the `3` backports. - [ ] If this issue requires less than `4` merge requests, post a message on the Security Release Tracking Issue and ping the Release Managers. diff --git a/.gitlab/issue_templates/actionable_insight.md b/.gitlab/issue_templates/actionable_insight.md index 08fbb30001c..7c65388eff4 100644 --- a/.gitlab/issue_templates/actionable_insight.md +++ b/.gitlab/issue_templates/actionable_insight.md @@ -12,7 +12,7 @@ Actionable insights always have a follow-up action that needs to take place as a #### Description -- [ ] Provide some brief detials on the actionable insight and the action to take +- [ ] Provide some brief details on the actionable insight and the action to take ------------------------------------------------------------------------------- @@ -28,4 +28,4 @@ Actionable insights always have a follow-up action that needs to take place as a - ~"Actionable Insight" + /label ~"Actionable Insight" diff --git a/.gitlab/merge_request_templates/Documentation.md b/.gitlab/merge_request_templates/Documentation.md index fb828b995b1..b17043fd3b9 100644 --- a/.gitlab/merge_request_templates/Documentation.md +++ b/.gitlab/merge_request_templates/Documentation.md @@ -16,9 +16,10 @@ ## Author's checklist (required) - [ ] Follow the [Documentation Guidelines](https://docs.gitlab.com/ee/development/documentation/) and [Style Guide](https://docs.gitlab.com/ee/development/documentation/styleguide.html). -- If you have `developer` access or higher (for example, GitLab team members or [Core Team](https://about.gitlab.com/community/core-team/) members) +- If you have **Developer** permissions or higher: + - [ ] Ensure that the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide.html#product-badges) is added to doc's `h1`. - [ ] Apply the ~documentation label, plus: - - The corresponding DevOps stage and group label, if applicable. + - The corresponding DevOps stage and group labels, if applicable. - ~"development guidelines" when changing docs under `doc/development/*`, `CONTRIBUTING.md`, or `README.md`. - ~"development guidelines" and ~"Documentation guidelines" when changing docs under `development/documentation/*`. - ~"development guidelines" and ~"Description templates (.gitlab/\*)" when creating/updating issue and MR description templates. @@ -30,10 +31,9 @@ When applicable: - [ ] Update the [permissions table](https://docs.gitlab.com/ee/user/permissions.html). - [ ] Link docs to and from the higher-level index page, plus other related docs where helpful. +- [ ] Add the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide.html#product-badges) accordingly. - [ ] Add [GitLab's version history note(s)](https://docs.gitlab.com/ee/development/documentation/styleguide.html#text-for-documentation-requiring-version-text). -- [ ] Add the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide.html#product-badges). - [ ] Add/update the [feature flag section](https://docs.gitlab.com/ee/development/documentation/feature_flags.html). -- [ ] If you're changing document headings, search `doc/*`, `app/views/*`, and `ee/app/views/*` for old headings replacing with the new ones to [avoid broken anchors](https://docs.gitlab.com/ee/development/documentation/styleguide.html#anchor-links). ## Review checklist @@ -46,8 +46,9 @@ All reviewers can help ensure accuracy, clarity, completeness, and adherence to **2. Technical Writer** - [ ] Technical writer review. If not requested for this MR, must be scheduled post-merge. To request for this MR, assign the writer listed for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/product-categories/#devops-stages). - - [ ] Ensure ~"Technical Writing", ~"documentation", and a `docs::` scoped label are added. - - [ ] Add ~docs-only when the only files changed are under `doc/*`. + - [ ] Ensure docs metadata are present and up-to-date. + - [ ] Ensure ~"Technical Writing" and ~"documentation" are added. + - [ ] Add the corresponding `docs::` scoped label. - [ ] Add ~"tw::doing" when starting work on the MR. - [ ] Add ~"tw::finished" if Technical Writing team work on the MR is complete but it remains open. diff --git a/.gitlab/merge_request_templates/Security Release.md b/.gitlab/merge_request_templates/Security Release.md index bdf26041e62..eda16747c13 100644 --- a/.gitlab/merge_request_templates/Security Release.md +++ b/.gitlab/merge_request_templates/Security Release.md @@ -23,7 +23,6 @@ See [the general developer security release guidelines](https://gitlab.com/gitla - [ ] Ensure it's approved by an AppSec engineer. - If you're unsure who should approve, find the AppSec engineer associated to the issue in the [Canonical repository], or ask #sec-appsec on Slack. - Trigger the [`package-and-qa` build]. The docker image generated will be used by the AppSec engineer to validate the security vulnerability has been remediated. - - [ ] Merge request _must_ close the corresponding security issue. - [ ] For a backport MR targeting a versioned stable branch (`X-Y-stable-ee`) - [ ] Ensure it's approved by a maintainer. |