diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 08:43:02 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 08:43:02 +0000 |
| commit | d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb (patch) | |
| tree | 2341ef426af70ad1e289c38036737e04b0aa5007 /.gitlab/ci | |
| parent | d6e514dd13db8947884cd58fe2a9c2a063400a9b (diff) | |
| download | gitlab-ce-d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42
Diffstat (limited to '.gitlab/ci')
| -rw-r--r-- | .gitlab/ci/build-images.gitlab-ci.yml | 3 | ||||
| -rw-r--r-- | .gitlab/ci/dast.gitlab-ci.yml | 205 | ||||
| -rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 14 | ||||
| -rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 73 | ||||
| -rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 3 | ||||
| -rw-r--r-- | .gitlab/ci/memory.gitlab-ci.yml | 34 | ||||
| -rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 35 | ||||
| -rw-r--r-- | .gitlab/ci/review-apps/dast.gitlab-ci.yml | 191 | ||||
| -rw-r--r-- | .gitlab/ci/review-apps/main.gitlab-ci.yml | 106 | ||||
| -rw-r--r-- | .gitlab/ci/review-apps/qa.gitlab-ci.yml | 128 | ||||
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 230 | ||||
| -rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 397 | ||||
| -rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 16 | ||||
| -rw-r--r-- | .gitlab/ci/static-analysis.gitlab-ci.yml | 11 | ||||
| -rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 3 |
15 files changed, 814 insertions, 635 deletions
diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index 0169f017063..6a222d8937f 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -28,7 +28,8 @@ build-qa-image: script: - !reference [.base-image-build, script] - echo $QA_IMAGE - - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true + - echo $QA_IMAGE_BRANCH + - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --destination=${QA_IMAGE_BRANCH} --cache=true # This image is used by: # - The `CNG` pipelines (via the `review-build-cng` job): https://gitlab.com/gitlab-org/build/CNG/-/blob/cfc67136d711e1c8c409bf8e57427a644393da2f/.gitlab-ci.yml#L335 diff --git a/.gitlab/ci/dast.gitlab-ci.yml b/.gitlab/ci/dast.gitlab-ci.yml deleted file mode 100644 index 309714f8739..00000000000 --- a/.gitlab/ci/dast.gitlab-ci.yml +++ /dev/null @@ -1,205 +0,0 @@ -.dast_conf: - tags: - - prm - # For scheduling dast job - extends: - - .reports:rules:schedule-dast - image: - name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" - resource_group: dast_scan - variables: - DAST_USERNAME_FIELD: "user[login]" - DAST_PASSWORD_FIELD: "user[password]" - DAST_FULL_SCAN_ENABLED: "true" - DAST_SPIDER_MINS: 0 - # TBD pin to a version - DAST_VERSION: 1.22.1 - # -Xmx is used to set the JVM memory to 6GB to prevent DAST OutOfMemoryError. - DAST_ZAP_CLI_OPTIONS: "-Xmx6144m" - DAST_RULES: "41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,90019,10109,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028,10003,50003,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90020,90022,90033" - before_script: - - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' - - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' - - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' - # Below three lines can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/230687 is fixed - - mkdir -p /zap/xml - - 'sed -i "84 s/true/false/" /zap/xml/config.xml' - - cat /zap/xml/config.xml - # Help pages are excluded from scan as they are static pages. - # profile/two_factor_auth is excluded from scan to prevent 2FA from being turned on from user profile, which will reduce coverage. - - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/help/.*,${DAST_WEBSITE}/profile/two_factor_auth,${DAST_WEBSITE}/users/sign_out"' - # Exclude the automatically generated monitoring project from being tested due to https://gitlab.com/gitlab-org/gitlab/-/issues/260362 - - 'DAST_AUTH_EXCLUDE_URLS="${DAST_AUTH_EXCLUDE_URLS},https://.*\.gitlab-review\.app/gitlab-instance-(administrators-)?[a-zA-Z0-9]{8}/.*"' - - enable_rule () { read all_rules; rule=$1; echo $all_rules | sed -r "s/(,)?$rule(,)?/\1-1\2/" ; } - # Sort ids in DAST_RULES ascendingly, which is required when using DAST_RULES as argument to enable_rule - - 'DAST_RULES=$(echo $DAST_RULES | tr "," "\n" | sort -n | paste -sd ",")' - needs: ["review-deploy"] - stage: dast - # Default job timeout set to 90m and dast rules needs 2h to so that it won't timeout. - timeout: 2h - # Add retry because of intermittent connection problems. See https://gitlab.com/gitlab-org/gitlab/-/issues/244313 - retry: 1 - artifacts: - paths: - - gl-dast-report.json # GitLab-specific - reports: - dast: gl-dast-report.json - expire_in: 1 week # GitLab-specific - -# DAST scan with a subset of Release scan rules. -DAST-fullscan-ruleset1: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user1" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10019 | enable_rule 10020 | enable_rule 10021 | enable_rule 10023 | enable_rule 10024 | enable_rule 10025 | enable_rule 10037 | enable_rule 10040 | enable_rule 10054 | enable_rule 10055 | enable_rule 10056) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with a subset of Release scan rules. -DAST-fullscan-ruleset2: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user2" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90011 | enable_rule 90020 | enable_rule 90022 | enable_rule 90033) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with a subset of Release scan rules. -DAST-fullscan-ruleset3: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user3" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40016 | enable_rule 40017 | enable_rule 50000 | enable_rule 50001) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with a subset of Release scan rules. -DAST-fullscan-ruleset4: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user4" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 0 | enable_rule 2 | enable_rule 3 | enable_rule 7 ) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with a subset of Release scan rules. -DAST-fullscan-ruleset5: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user5" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10010 | enable_rule 10011 | enable_rule 10017 | enable_rule 10019) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with a subset of Release scan rules. -DAST-fullscan-ruleset6: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user6" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 30001 | enable_rule 40009) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed -# DAST scan with a subset of Beta scan rules. -# DAST-fullscan-ruleset7: -# extends: -# - .dast_conf -# variables: -# DAST_USERNAME: "user7" -# script: -# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10098 | enable_rule 10105 | enable_rule 10202 | enable_rule 30002 | enable_rule 40003 | enable_rule 40008 | enable_rule 40009) -# - echo $DAST_EXCLUDE_RULES -# - /analyze -t $DAST_WEBSITE -d - -# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed -# Below jobs runs DAST scans with one time consuming scan rule. These scan rules are disabled in above jobs so that those jobs won't timeout. -# DAST scan with rule - 20019 External Redirect -# DAST-fullscan-rule-20019: -# extends: -# - .dast_conf -# variables: -# DAST_USERNAME: "user8" -# script: -# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 20019) -# - echo $DAST_EXCLUDE_RULES -# - /analyze -t $DAST_WEBSITE -d - -# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed -# DAST scan with rule - 10107 Httpoxy - Proxy Header Misuse - Active/beta -# DAST-fullscan-rule-10107: -# extends: -# - .dast_conf -# variables: -# DAST_USERNAME: "user9" -# script: -# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10107) -# - echo $DAST_EXCLUDE_RULES -# - /analyze -t $DAST_WEBSITE -d - -# DAST scan with rule - 90020 Remote OS Command Injection -DAST-fullscan-rule-90020: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user10" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90020) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with rule - 40018 SQL Injection - Active/release -DAST-fullscan-rule-40018: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user11" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40018) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with rule - 40014 Cross Site Scripting (Persistent) - Active/release -DAST-fullscan-rule-40014: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user12" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40014) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with rule - 6 Path travesal -DAST-fullscan-rule-6: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user13" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 6) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d - -# DAST scan with rule - 40012 Cross Site Scripting (Reflected) -DAST-fullscan-rule-40012: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user14" - script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40012) - - echo $DAST_EXCLUDE_RULES - - /analyze -t $DAST_WEBSITE -d diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index c585047f916..f4d8698f22d 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -75,17 +75,3 @@ ui-docs-links lint: needs: [] script: - bundle exec haml-lint -i DocumentationLinks - -deprecations-doc check: - variables: - SETUP_DB: "false" - extends: - - .default-retry - - .rails-cache - - .default-before_script - - .docs:rules:deprecations - stage: test - needs: [] - script: - - bundle exec rake gitlab:docs:check_deprecations - allow_failure: true diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 48f85219ff4..6974d63a49c 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -71,6 +71,12 @@ compile-test-assets as-if-foss: - .frontend:rules:compile-test-assets-as-if-foss - .as-if-foss +compile-test-assets as-if-jh: + extends: + - compile-test-assets + - .frontend:rules:compile-test-assets-as-if-jh + needs: ["add-jh-folder"] + update-assets-compile-production-cache: extends: - compile-production-assets @@ -112,7 +118,7 @@ update-storybook-yarn-cache: - .rails-cache - .use-pg12 stage: fixtures - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"] + needs: ["setup-test-env", "retrieve-tests-metadata"] variables: WEBPACK_VENDOR_DLL: "true" script: @@ -128,23 +134,38 @@ update-storybook-yarn-cache: - tmp/tests/frontend/ - knapsack/ -rspec frontend_fixture: +# Builds FOSS, and EE fixtures in the EE project. +# Builds FOSS fixtures in the FOSS project. +rspec-all frontend_fixture: extends: - .frontend-fixtures-base - .frontend:rules:default-frontend-jobs - parallel: 2 + needs: + - !reference [.frontend-fixtures-base, needs] + - "compile-test-assets" + parallel: 5 -rspec frontend_fixture as-if-foss: +# Builds FOSS fixtures in the EE project, with the `ee/` folder removed (due to `as-if-foss`). +rspec-all frontend_fixture as-if-foss: extends: - .frontend-fixtures-base - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss + needs: + - !reference [.frontend-fixtures-base, needs] + - "compile-test-assets as-if-foss" -rspec-ee frontend_fixture: +# Builds FOSS, EE, and JH fixtures in the EE project, with the `jh/` folder added (due to `as-if-jh`). +rspec-all frontend_fixture as-if-jh: extends: - .frontend-fixtures-base - - .frontend:rules:default-frontend-jobs-ee - parallel: 3 + - .frontend:rules:default-frontend-jobs-as-if-jh + needs: + - !reference [.frontend-fixtures-base, needs] + - "compile-test-assets as-if-jh" + - "add-jh-folder" + script: + - echo "This job is currently doing nothing since there's no specific JH fixtures yet. To enable this job, remove this line." graphql-schema-dump: variables: @@ -172,7 +193,9 @@ graphql-schema-dump: # Disable warnings in browserslist which can break on backports # https://github.com/browserslist/browserslist/blob/a287ec6/node.js#L367-L384 BROWSERSLIST_IGNORE_OLD_DATA: "true" + SETUP_DB: "false" before_script: + - !reference [.default-before_script, before_script] - *yarn-install stage: test @@ -194,11 +217,7 @@ jest: extends: - .jest-base - .frontend:rules:jest - needs: - - job: "detect-tests" - - job: "rspec frontend_fixture" - - job: "rspec-ee frontend_fixture" - optional: true + needs: ["rspec-all frontend_fixture"] artifacts: name: coverage-frontend expire_in: 31d @@ -215,6 +234,9 @@ jest minimal: extends: - jest - .frontend:rules:jest:minimal + needs: + - !reference [jest, needs] + - "detect-tests" script: - run_timed_command "yarn jest:ci:minimal" @@ -225,9 +247,7 @@ jest-integration: script: - run_timed_command "yarn jest:integration --ci" needs: - - job: "rspec frontend_fixture" - - job: "rspec-ee frontend_fixture" - optional: true + - job: "rspec-all frontend_fixture" - job: "graphql-schema-dump" jest-as-if-foss: @@ -235,9 +255,17 @@ jest-as-if-foss: - .jest-base - .frontend:rules:default-frontend-jobs-as-if-foss - .as-if-foss - needs: ["rspec frontend_fixture as-if-foss"] + needs: ["rspec-all frontend_fixture as-if-foss"] parallel: 2 +jest-as-if-jh: + extends: + - .jest-base + - .frontend:rules:default-frontend-jobs-as-if-jh + needs: ["rspec-all frontend_fixture as-if-jh", "add-jh-folder"] + script: + - echo "This job is currently doing nothing since there's no specific JH Jest tests yet. To enable this job, remove this line." + coverage-frontend: extends: - .default-retry @@ -341,9 +369,7 @@ startup-css-check: - .frontend:rules:default-frontend-jobs needs: - job: "compile-test-assets" - - job: "rspec frontend_fixture" - - job: "rspec-ee frontend_fixture" - optional: true + - job: "rspec-all frontend_fixture" startup-css-check as-if-foss: extends: @@ -352,7 +378,7 @@ startup-css-check as-if-foss: - .frontend:rules:default-frontend-jobs-as-if-foss needs: - job: "compile-test-assets as-if-foss" - - job: "rspec frontend_fixture as-if-foss" + - job: "rspec-all frontend_fixture as-if-foss" .compile-storybook-base: extends: @@ -361,11 +387,15 @@ startup-css-check as-if-foss: script: - *storybook-yarn-install - yarn run storybook:build + needs: ["graphql-schema-dump"] compile-storybook: extends: - .compile-storybook-base - .frontend:rules:default-frontend-jobs + needs: + - !reference [.compile-storybook-base, needs] + - job: "rspec-all frontend_fixture" artifacts: name: storybook expire_in: 31d @@ -378,3 +408,6 @@ compile-storybook as-if-foss: - .compile-storybook-base - .as-if-foss - .frontend:rules:default-frontend-jobs-as-if-foss + needs: + - !reference [.compile-storybook-base, needs] + - job: "rspec-all frontend_fixture as-if-foss" diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index d9978a44ffb..d0c26d60066 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -10,6 +10,7 @@ .default-before_script: before_script: + - echo $FOSS_ONLY - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' - export GOPATH=$CI_PROJECT_DIR/.go - mkdir -p $GOPATH @@ -193,10 +194,12 @@ .storybook-yarn-cache: cache: + - *node-modules-cache - *storybook-node-modules-cache .storybook-yarn-cache-push: cache: + - *node-modules-cache # We don't push this cache as it's already rebuilt by `update-yarn-cache` - *storybook-node-modules-cache-push .use-pg11: diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index f3ad8f81da5..9234b116ff8 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -4,6 +4,12 @@ - .rails-cache - .default-before_script - .memory:rules + variables: + METRICS_FILE: "metrics.txt" + artifacts: + reports: + metrics: "${METRICS_FILE}" + expire_in: 31d memory-static: extends: .only-code-memory-job-base @@ -11,24 +17,25 @@ memory-static: needs: ["setup-test-env"] variables: SETUP_DB: "false" + MEMORY_BUNDLE_MEM_FILE: "tmp/memory_bundle_mem.txt" + MEMORY_BUNDLE_OBJECTS_FILE: "tmp/memory_bundle_objects.txt" script: # Uses two different reports from the 'derailed_benchmars' gem. # Loads each of gems in the Gemfile and checks how much memory they consume when they are required. # 'derailed_benchmarks' internally uses 'get_process_mem' - - bundle exec derailed bundle:mem > tmp/memory_bundle_mem.txt - - scripts/generate-gems-size-metrics-static tmp/memory_bundle_mem.txt >> 'tmp/memory_metrics.txt' + - bundle exec derailed bundle:mem > "${MEMORY_BUNDLE_MEM_FILE}" + - scripts/generate-gems-size-metrics-static "${MEMORY_BUNDLE_MEM_FILE}" >> "${METRICS_FILE}" # Outputs detailed information about objects created while gems are loaded. # 'derailed_benchmarks' internally uses 'memory_profiler' - - bundle exec derailed bundle:objects > tmp/memory_bundle_objects.txt - - scripts/generate-gems-memory-metrics-static tmp/memory_bundle_objects.txt >> 'tmp/memory_metrics.txt' + - bundle exec derailed bundle:objects > "${MEMORY_BUNDLE_OBJECTS_FILE}" + - scripts/generate-gems-memory-metrics-static "${MEMORY_BUNDLE_OBJECTS_FILE}" >> "${METRICS_FILE}" artifacts: paths: - - tmp/memory_*.txt - reports: - metrics: tmp/memory_metrics.txt - expire_in: 31d + - "${METRICS_FILE}" + - "${MEMORY_BUNDLE_MEM_FILE}" + - "${MEMORY_BUNDLE_OBJECTS_FILE}" # Show memory usage caused by invoking require per gem. # Unlike `memory-static`, it hits the app with one request to ensure that any last minute require-s have been called. @@ -44,12 +51,11 @@ memory-on-boot: NODE_ENV: "production" RAILS_ENV: "production" SETUP_DB: "true" + MEMORY_ON_BOOT_FILE: "tmp/memory_on_boot.txt" script: - - PATH_TO_HIT="/users/sign_in" CUT_OFF=0.3 bundle exec derailed exec perf:mem >> 'tmp/memory_on_boot.txt' - - scripts/generate-memory-metrics-on-boot tmp/memory_on_boot.txt >> 'tmp/memory_on_boot_metrics.txt' + - PATH_TO_HIT="/users/sign_in" CUT_OFF=0.3 bundle exec derailed exec perf:mem >> "${MEMORY_ON_BOOT_FILE}" + - scripts/generate-memory-metrics-on-boot "${MEMORY_ON_BOOT_FILE}" >> "${METRICS_FILE}" artifacts: paths: - - tmp/memory_*.txt - reports: - metrics: tmp/memory_on_boot_metrics.txt - expire_in: 31d + - "${METRICS_FILE}" + - "${MEMORY_ON_BOOT_FILE}" diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index a5403073e1b..b581cf83d56 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -1,7 +1,7 @@ include: - template: Jobs/Code-Quality.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml + - template: Jobs/SAST.gitlab-ci.yml + - template: Jobs/Secret-Detection.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml @@ -13,6 +13,7 @@ code_quality: paths: - gl-code-quality-report.json # GitLab-specific rules: !reference [".reports:rules:code_quality", rules] + allow_failure: true .sast-analyzer: # We need to re-`extends` from `sast` as the `extends` here overrides the one from the template. @@ -27,16 +28,13 @@ code_quality: variables: SAST_BRAKEMAN_LEVEL: 2 # GitLab-specific SAST_EXCLUDED_PATHS: "qa, spec, doc, ee/spec, config/gitlab.yml.example, tmp" # GitLab-specific - SAST_EXCLUDED_ANALYZERS: bandit, flawfinder, phpcs-security-audit, pmd-apex, security-code-scan, spotbugs, eslint + SAST_EXCLUDED_ANALYZERS: bandit, flawfinder, phpcs-security-audit, pmd-apex, security-code-scan, spotbugs, eslint, nodejs-scan brakeman-sast: - rules: !reference [".reports:rules:sast", rules] - -nodejs-scan-sast: - rules: !reference [".reports:rules:sast", rules] + rules: !reference [".reports:rules:brakeman-sast", rules] semgrep-sast: - rules: !reference [".reports:rules:sast", rules] + rules: !reference [".reports:rules:semgrep-sast", rules] gosec-sast: variables: @@ -52,7 +50,7 @@ gosec-sast: cache: paths: - vendor/go - rules: !reference [".reports:rules:sast", rules] + rules: !reference [".reports:rules:gosec-sast", rules] .secret-analyzer: extends: .default-retry @@ -73,6 +71,7 @@ secret_detection: needs: [] variables: DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp" # GitLab-specific + DS_EXCLUDED_ANALYZERS: "gemnasium-maven" artifacts: paths: - gl-dependency-scanning-report.json # GitLab-specific @@ -82,11 +81,6 @@ gemnasium-dependency_scanning: before_script: # git-lfs is needed for auto-remediation - apk add git-lfs - after_script: - # Post-processing - - apk add jq - # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390 - - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json rules: !reference [".reports:rules:gemnasium-dependency_scanning", rules] bundler-audit-dependency_scanning: @@ -101,8 +95,7 @@ gemnasium-python-dependency_scanning: # Analyze dependencies for malicious behavior # See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter .package_hunter-base: - extends: - - .default-retry + extends: .default-retry stage: test image: name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:1.1.0 @@ -116,6 +109,8 @@ gemnasium-python-dependency_scanning: before_script: - rm -r spec locale .git app/assets/images doc/ - cd .. && tar -I "gzip --best" -cf gitlab.tgz gitlab/ + script: + - node /usr/src/app/cli.js analyze --format gitlab --manager ${PACKAGE_MANAGER} gitlab.tgz | tee ${CI_PROJECT_DIR}/gl-dependency-scanning-report.json artifacts: paths: - gl-dependency-scanning-report.json @@ -127,15 +122,15 @@ package_hunter-yarn: extends: - .package_hunter-base - .reports:rules:package_hunter-yarn - script: - - node /usr/src/app/cli.js analyze --format gitlab --manager yarn gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json + variables: + PACKAGE_MANAGER: yarn package_hunter-bundler: extends: - .package_hunter-base - .reports:rules:package_hunter-bundler - script: - - node /usr/src/app/cli.js analyze --format gitlab --manager bundler gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json + variables: + PACKAGE_MANAGER: bundler license_scanning: extends: .default-retry diff --git a/.gitlab/ci/review-apps/dast.gitlab-ci.yml b/.gitlab/ci/review-apps/dast.gitlab-ci.yml new file mode 100644 index 00000000000..512c850b7da --- /dev/null +++ b/.gitlab/ci/review-apps/dast.gitlab-ci.yml @@ -0,0 +1,191 @@ +.dast_conf: + tags: + - prm + # For scheduling dast job + extends: + - .reports:rules:schedule-dast + image: + name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" + resource_group: dast_scan + variables: + DAST_USERNAME_FIELD: "user[login]" + DAST_PASSWORD_FIELD: "user[password]" + DAST_SUBMIT_FIELD: "commit" + DAST_FULL_SCAN_ENABLED: "true" + DAST_VERSION: 2 + GIT_STRATEGY: none + # -Xmx is used to set the JVM memory to 6GB to prevent DAST OutOfMemoryError. + DAST_ZAP_CLI_OPTIONS: "-Xmx6144m" + before_script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # Help pages are excluded from scan as they are static pages. + # profile/two_factor_auth is excluded from scan to prevent 2FA from being turned on from user profile, which will reduce coverage. + - 'DAST_EXCLUDE_URLS="${DAST_WEBSITE}/help/.*,${DAST_WEBSITE}/-/profile/two_factor_auth,${DAST_WEBSITE}/users/sign_out"' + # Exclude the automatically generated monitoring project from being tested due to https://gitlab.com/gitlab-org/gitlab/-/issues/260362 + - 'export DAST_EXCLUDE_URLS="${DAST_EXCLUDE_URLS},${DAST_WEBSITE}/gitlab-instance-.*"' + needs: ["review-deploy"] + stage: dast + # Default job timeout set to 90m and dast rules needs 2h to so that it won't timeout. + timeout: 2h + # Add retry because of intermittent connection problems. See https://gitlab.com/gitlab-org/gitlab/-/issues/244313 + retry: 1 + artifacts: + paths: + - gl-dast-report.json # GitLab-specific + reports: + dast: gl-dast-report.json + expire_in: 1 week # GitLab-specific + allow_failure: true + +# DAST scan with a subset of Release scan rules. +# ZAP rule details can be found at https://www.zaproxy.org/docs/alerts/ + +# 10019, 10021 Missing security headers +# 10023, 10024, 10025, 10037 Information Disclosure +# 10040 Secure Pages Include Mixed Content +# 10055 CSP +# 10056 X-Debug-Token Information Leak +# Duration: 14 minutes 20 seconds + +dast:secureHeaders-csp-infoLeak: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user1" + DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10055,10056" + script: + - /analyze + +# 90023 XML External Entity Attack +# Duration: 41 minutes 20 seconds +# 90019 Server Side Code Injection +# Duration: 34 minutes 31 seconds +dast:XXE-SrvSideInj: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user2" + DAST_ONLY_INCLUDE_RULES: "90023,90019" + script: + - /analyze + +# 0 Directory Browsing +# 2 Private IP Disclosure +# 3 Session ID in URL Rewrite +# 7 Remote File Inclusion +# Duration: 63 minutes 43 seconds +# 90034 Cloud Metadata Potentially Exposed +# Duration: 13 minutes 48 seconds +# 90022 Application Error Disclosure +# Duration: 12 minutes 7 seconds +dast:infoLeak-fileInc-DirBrowsing: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user3" + DAST_ONLY_INCLUDE_RULES: "0,2,3,7,90034,90022" + script: + - /analyze + +# 10010 Cookie No HttpOnly Flag +# 10011 Cookie Without Secure Flag +# 10017 Cross-Domain JavaScript Source File Inclusion +# 10029 Cookie Poisoning +# 90033 Loosely Scoped Cookie +# 10054 Cookie Without SameSite Attribute +# Duration: 13 minutes 23 seconds +dast:insecureCookie: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user4" + DAST_ONLY_INCLUDE_RULES: "10010,10011,10017,10029,90033,10054" + script: + - /analyze + + +# 20012 Anti-CSRF Tokens Check +# 10202 Absence of Anti-CSRF Tokens +# https://gitlab.com/gitlab-com/gl-security/appsec/appsec-team/-/issues/192 + +# Commented because of lot of FP's +# dast:csrfTokenCheck: +# extends: +# - .dast_conf +# variables: +# DAST_USERNAME: "user6" +# DAST_ONLY_INCLUDE_RULES: "20012,10202" +# script: +# - /analyze + +# 10098 Cross-Domain Misconfiguration +# 10105 Weak Authentication Method +# 40003 CRLF Injection +# 40008 Parameter Tampering +# Duration: 71 minutes 15 seconds +dast:corsMisconfig-weakauth-crlfInj: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user5" + DAST_ONLY_INCLUDE_RULES: "10098,10105,40003,40008" + script: + - /analyze + +# 20019 External Redirect +# 20014 HTTP Parameter Pollution +# Duration: 46 minutes 12 seconds +dast:extRedirect-paramPollution: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user6" + DAST_ONLY_INCLUDE_RULES: "20019,20014" + script: + - /analyze + +# 40022 SQL Injection - PostgreSQL +# Duration: 53 minutes 59 seconds +dast:sqlInjection: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user7" + DAST_ONLY_INCLUDE_RULES: "40022" + script: + - /analyze + +# 40014 Cross Site Scripting (Persistent) +# Duration: 21 minutes 50 seconds +dast:xss-persistent: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user8" + DAST_ONLY_INCLUDE_RULES: "40014" + script: + - /analyze + +# 40012 Cross Site Scripting (Reflected) +# Duration: 73 minutes 15 seconds +dast:xss-reflected: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user9" + DAST_ONLY_INCLUDE_RULES: "40012" + script: + - /analyze + +# 40013 Session Fixation +# Duration: 44 minutes 25 seconds +dast:sessionFixation: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user10" + DAST_ONLY_INCLUDE_RULES: "40013" + script: + - /analyze diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml new file mode 100644 index 00000000000..6fe9e39cb82 --- /dev/null +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -0,0 +1,106 @@ +stages: + - prepare + - deploy + - qa + - post-qa + - dast + +include: + - local: .gitlab/ci/global.gitlab-ci.yml + - local: .gitlab/ci/rules.gitlab-ci.yml + - local: .gitlab/ci/review-apps/qa.gitlab-ci.yml + - local: .gitlab/ci/review-apps/dast.gitlab-ci.yml + +.base-before_script: &base-before_script + - source ./scripts/utils.sh + - source ./scripts/review_apps/review-apps.sh + - install_api_client_dependencies_with_apk + +review-build-cng: + extends: + - .default-retry + - .review:rules:review-build-cng + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 + stage: prepare + variables: + CNG_PROJECT_ACCESS_TOKEN: "${CNG_MIRROR_PROJECT_ACCESS_TOKEN}" # "Multi-pipeline (from 'gitlab-org/gitlab' 'review-build-cng' job)" at https://gitlab.com/gitlab-org/build/CNG-mirror/-/settings/access_tokens + CNG_PROJECT_PATH: "gitlab-org/build/CNG-mirror" + before_script: + - source ./scripts/utils.sh + - install_gitlab_gem + script: + - ./scripts/trigger-build cng + +.review-workflow-base: + extends: + - .default-retry + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3.5-kubectl1.17 + variables: + HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" + DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" + GITLAB_HELM_CHART_REF: "v5.2.1" + environment: + name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} + url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} + on_stop: review-stop + auto_stop_in: 48 hours + +review-deploy: + extends: + - .review-workflow-base + - .review:rules:review-deploy + stage: deploy + needs: ["review-build-cng"] + resource_group: "review/${CI_COMMIT_REF_NAME}" + before_script: + - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) + - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) + - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) + - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt + - *base-before_script + script: + - check_kube_domain + - download_chart + - date + - deploy || (display_deployment_debug && exit 1) + - verify_deploy || exit 1 + - disable_sign_ups || (delete_release && exit 1) + after_script: + # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. + # Set DAST_RUN to true when jobs are manually scheduled. + - if [ "$DAST_RUN" == "true" ]; then source scripts/review_apps/seed-dast-test-data.sh; TRACE=1 trigger_proj_user_creation; fi + artifacts: + paths: + - environment_url.txt + - curl_output.txt + expire_in: 7 days + when: always + +.review-stop-base: + extends: .review-workflow-base + environment: + action: stop + dependencies: [] + variables: + # We're cloning the repo instead of downloading the script for now + # because some repos are private and CI_JOB_TOKEN cannot access files. + # See https://gitlab.com/gitlab-org/gitlab/issues/191273 + GIT_DEPTH: 1 + before_script: + - *base-before_script + +review-delete-deployment: + extends: + - .review-stop-base + - .review:rules:review-delete-deployment + stage: prepare + script: + - delete_release + +review-stop: + extends: + - .review-stop-base + - .review:rules:review-stop + stage: post-qa + script: + - delete_k8s_release_namespace diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml new file mode 100644 index 00000000000..6b9d4feb3c8 --- /dev/null +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -0,0 +1,128 @@ +.review-qa-base: + extends: + - .use-docker-in-docker + image: + name: ${QA_IMAGE} + entrypoint: [""] + stage: qa + needs: ["review-deploy"] + variables: + QA_DEBUG: "true" + QA_CAN_TEST_GIT_PROTOCOL_V2: "false" + QA_GENERATE_ALLURE_REPORT: "true" + GITLAB_USERNAME: "root" + GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITLAB_ADMIN_USERNAME: "root" + GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" + EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" + SIGNUP_DISABLED: "true" + before_script: + # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + fi + - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" + - echo "${CI_ENVIRONMENT_URL}" + - cd qa + artifacts: + paths: + - qa/tmp + expire_in: 7 days + when: always + +.allure-report-base: + image: + name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.6 + entrypoint: [""] + stage: post-qa + variables: + GIT_STRATEGY: none + STORAGE_CREDENTIALS: $QA_ALLURE_REPORT_GCS_CREDENTIALS + GITLAB_AUTH_TOKEN: $GITLAB_QA_MR_ALLURE_REPORT_TOKEN + ALLURE_PROJECT_PATH: $CI_PROJECT_PATH + ALLURE_MERGE_REQUEST_IID: $CI_MERGE_REQUEST_IID + allow_failure: true + script: + - | + allure-report-publisher upload gcs \ + --results-glob="qa/tmp/allure-results/*" \ + --bucket="gitlab-qa-allure-reports" \ + --prefix="$ALLURE_REPORT_PATH_PREFIX/$CI_COMMIT_REF_SLUG" \ + --update-pr="comment" \ + --copy-latest \ + --ignore-missing-results \ + --color + +review-qa-smoke: + extends: + - .review-qa-base + - .review:rules:review-qa-smoke + retry: 1 # This is confusing but this means "2 runs at max". + variables: + QA_RUN_TYPE: review-qa-smoke + script: + - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}" + +review-qa-all: + extends: + - .review-qa-base + - .review:rules:review-qa-all + variables: + QA_RUN_TYPE: review-qa-all + parallel: 5 + script: + - export KNAPSACK_REPORT_PATH=knapsack/master_report.json + - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb + - | + bin/test Test::Instance::All "${CI_ENVIRONMENT_URL}" \ + -- \ + --color --format documentation \ + --format RspecJunitFormatter --out tmp/rspec.xml + artifacts: + reports: + junit: qa/tmp/rspec.xml + +review-performance: + extends: + - .default-retry + - .review:rules:review-performance + image: + name: sitespeedio/sitespeed.io + entrypoint: [""] + stage: qa + needs: ["review-deploy"] + before_script: + - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" + - echo "${CI_ENVIRONMENT_URL}" + - mkdir -p gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - mkdir -p sitespeed-results + script: + - /start.sh --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}" + after_script: + - mv sitespeed-results/data/performance.json performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + performance: performance.json + expire_in: 31d + +allure-report-qa-smoke: + extends: + - .allure-report-base + - .review:rules:review-qa-smoke-report + needs: ["review-qa-smoke"] + variables: + ALLURE_REPORT_PATH_PREFIX: gitlab-review-smoke + ALLURE_JOB_NAME: review-qa-smoke + +allure-report-qa-all: + extends: + - .allure-report-base + - .review:rules:review-qa-all-report + needs: ["review-qa-all"] + variables: + ALLURE_REPORT_PATH_PREFIX: gitlab-review-all + ALLURE_JOB_NAME: review-qa-all diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index f20f3276867..b2b8c456ae2 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -16,225 +16,25 @@ review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb - gcp_cleanup -.base-before_script: &base-before_script - - source ./scripts/utils.sh - - source ./scripts/review_apps/review-apps.sh - - install_api_client_dependencies_with_apk - -review-build-cng: +start-review-app-pipeline: extends: - - .default-retry - - .review:rules:review-build-cng - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 - stage: review-prepare + - .review:rules:review-app-pipeline + stage: review needs: - - job: compile-production-assets + - job: build-assets-image artifacts: false + - job: build-qa-image + artifacts: false + # These variables are set in the pipeline schedules. + # They need to be explicitly passed on to the child pipeline. + # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword variables: - CNG_PROJECT_ACCESS_TOKEN: "${CNG_MIRROR_PROJECT_ACCESS_TOKEN}" # "Multi-pipeline (from 'gitlab-org/gitlab' 'review-build-cng' job)" at https://gitlab.com/gitlab-org/build/CNG-mirror/-/settings/access_tokens - CNG_PROJECT_PATH: "gitlab-org/build/CNG-mirror" - before_script: - - source ./scripts/utils.sh - - install_gitlab_gem - script: - - ./scripts/trigger-build cng - -.review-workflow-base: - extends: - - .default-retry - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3.5-kubectl1.17 - variables: - HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" - DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v5.2.1" - environment: - name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} - url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} - on_stop: review-stop - auto_stop_in: 48 hours - -review-deploy: - extends: - - .review-workflow-base - - .review:rules:review-deploy - stage: review - needs: ["review-build-cng"] - resource_group: "review/${CI_COMMIT_REF_NAME}" - before_script: - - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) - - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt - - *base-before_script - script: - - check_kube_domain - - download_chart - - date - - deploy || (display_deployment_debug && exit 1) - - verify_deploy || exit 1 - - disable_sign_ups || (delete_release && exit 1) - after_script: - # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. - # Set DAST_RUN to true when jobs are manually scheduled. - - if [ "$DAST_RUN" == "true" ]; then source scripts/review_apps/seed-dast-test-data.sh; TRACE=1 trigger_proj_user_creation; fi - artifacts: - paths: - - environment_url.txt - - curl_output.txt - expire_in: 7 days - when: always - -.review-stop-base: - extends: .review-workflow-base - environment: - action: stop - dependencies: [] - variables: - # We're cloning the repo instead of downloading the script for now - # because some repos are private and CI_JOB_TOKEN cannot access files. - # See https://gitlab.com/gitlab-org/gitlab/issues/191273 - GIT_DEPTH: 1 - before_script: - - *base-before_script - -review-delete-deployment: - extends: - - .review-stop-base - - .review:rules:review-delete-deployment - stage: prepare - script: - - delete_release - -review-stop: - extends: - - .review-stop-base - - .review:rules:review-stop - stage: post-qa - script: - - delete_k8s_release_namespace - -.review-qa-base: - extends: - - .use-docker-in-docker - image: - name: ${QA_IMAGE} - entrypoint: [""] - stage: qa - needs: ["build-qa-image", "review-deploy"] - variables: - QA_DEBUG: "true" - QA_CAN_TEST_GIT_PROTOCOL_V2: "false" - QA_GENERATE_ALLURE_REPORT: "true" - GITLAB_USERNAME: "root" - GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" - GITLAB_ADMIN_USERNAME: "root" - GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" - GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" - EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" - SIGNUP_DISABLED: "true" - before_script: - # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. - - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then - git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; - fi - - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - - echo "${CI_ENVIRONMENT_URL}" - - cd qa - artifacts: - paths: - - qa/tmp - expire_in: 7 days - when: always - -.allure-report-base: - image: - name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.4 - entrypoint: [""] - stage: post-qa - variables: - GIT_STRATEGY: none - STORAGE_CREDENTIALS: $QA_ALLURE_REPORT_GCS_CREDENTIALS - GITLAB_AUTH_TOKEN: $GITLAB_QA_MR_ALLURE_REPORT_TOKEN - allow_failure: true - script: - - | - allure-report-publisher upload gcs \ - --results-glob="qa/tmp/allure-results/*" \ - --bucket="gitlab-qa-allure-reports" \ - --prefix="$ALLURE_REPORT_PATH_PREFIX/$CI_COMMIT_REF_SLUG" \ - --update-pr="comment" \ - --copy-latest \ - --ignore-missing-results \ - --color - -review-qa-smoke: - extends: - - .review-qa-base - - .review:rules:review-qa-smoke - retry: 1 # This is confusing but this means "2 runs at max". - script: - - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}" - -review-qa-all: - extends: - - .review-qa-base - - .review:rules:review-qa-all - parallel: 5 - script: - - export KNAPSACK_REPORT_PATH=knapsack/master_report.json - - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb - - | - bin/test Test::Instance::All "${CI_ENVIRONMENT_URL}" \ - -- \ - --color --format documentation \ - --format RspecJunitFormatter --out tmp/rspec.xml - artifacts: - reports: - junit: qa/tmp/rspec.xml - -review-performance: - extends: - - .default-retry - - .review:rules:review-performance - image: - name: sitespeedio/sitespeed.io - entrypoint: [""] - stage: qa - needs: ["review-deploy"] - before_script: - - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - - echo "${CI_ENVIRONMENT_URL}" - - mkdir -p gitlab-exporter - - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js - - mkdir -p sitespeed-results - script: - - /start.sh --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}" - after_script: - - mv sitespeed-results/data/performance.json performance.json - artifacts: - paths: - - sitespeed-results/ - reports: - performance: performance.json - expire_in: 31d - -allure-report-qa-smoke: - extends: - - .allure-report-base - - .review:rules:review-qa-smoke-report - needs: ["review-qa-smoke"] - variables: - ALLURE_REPORT_PATH_PREFIX: gitlab-review-smoke - ALLURE_JOB_NAME: review-qa-smoke - -allure-report-qa-all: - extends: - - .allure-report-base - - .review:rules:review-qa-all-report - needs: ["review-qa-all"] - variables: - ALLURE_REPORT_PATH_PREFIX: gitlab-review-all - ALLURE_JOB_NAME: review-qa-all + FREQUENCY: $FREQUENCY + DAST_RUN: $DAST_RUN + trigger: + include: + - local: .gitlab/ci/review-apps/main.gitlab-ci.yml + strategy: depend danger-review: extends: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index a4a932c7dd0..8ddcf9c2094 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -10,6 +10,9 @@ .if-not-foss: &if-not-foss if: '$CI_PROJECT_NAME != "gitlab-foss" && $CI_PROJECT_NAME != "gitlab-ce" && $CI_PROJECT_NAME != "gitlabhq"' +.if-jh: &if-jh + if: '$CI_PROJECT_PATH == "gitlab-jh/gitlab"' + .if-default-refs: &if-default-refs if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG || $FORCE_GITLAB_CI' @@ -37,19 +40,22 @@ .if-automated-merge-request: &if-automated-merge-request if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "release-tools/update-gitaly" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /stable-ee$/' -.if-merge-request-title-as-if-foss: &if-merge-request-title-as-if-foss +.if-merge-request-labels-as-if-foss: &if-merge-request-labels-as-if-foss if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-as-if-foss/' -.if-merge-request-title-update-caches: &if-merge-request-title-update-caches +.if-merge-request-labels-as-if-jh: &if-merge-request-labels-as-if-jh + if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-as-if-jh/' + +.if-merge-request-labels-update-caches: &if-merge-request-labels-update-caches if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:update-cache/' -.if-merge-request-title-run-all-rspec: &if-merge-request-title-run-all-rspec +.if-merge-request-labels-run-all-rspec: &if-merge-request-labels-run-all-rspec if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-all-rspec/' -.if-merge-request-title-run-all-jest: &if-merge-request-title-run-all-jest +.if-merge-request-labels-run-all-jest: &if-merge-request-labels-run-all-jest if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-all-jest/' -.if-merge-request-run-decomposed: &if-merge-request-run-decomposed +.if-merge-request-labels-run-decomposed: &if-merge-request-labels-run-decomposed if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-decomposed/' .if-security-merge-request: &if-security-merge-request @@ -67,15 +73,24 @@ .if-dot-com-gitlab-org-schedule: &if-dot-com-gitlab-org-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"' +.if-dot-com-gitlab-org-schedule-child-pipeline: &if-dot-com-gitlab-org-schedule-child-pipeline + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY' + .if-dot-com-ee-schedule: &if-dot-com-ee-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule"' +.if-dot-com-ee-schedule-child-pipeline: &if-dot-com-ee-schedule-child-pipeline + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY' + .if-dot-com-ee-2-hourly-schedule: &if-dot-com-ee-2-hourly-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' .if-dot-com-ee-nightly-schedule: &if-dot-com-ee-nightly-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "nightly"' +.if-dot-com-ee-nightly-schedule-child-pipeline: &if-dot-com-ee-nightly-schedule-child-pipeline + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY == "nightly"' + .if-cache-credentials-schedule: &if-cache-credentials-schedule if: '$CI_REPO_CACHE_CREDENTIALS && $CI_PIPELINE_SOURCE == "schedule"' @@ -91,13 +106,6 @@ .if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_COMMIT_TAG' - -.if-rspec-fail-fast-disabled: &if-rspec-fail-fast-disabled - if: '$RSPEC_FAIL_FAST_ENABLED != "true"' - -.if-rspec-fail-fast-skipped: &if-rspec-fail-fast-skipped - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:skip-rspec-fail-fast/' - # For Security merge requests, the gitlab-release-tools-bot triggers a new # pipeline for the "Pipelines for merged results" feature. If the pipeline # fails, we notify release managers. @@ -120,6 +128,7 @@ - ".gitlab/ci/frontend.gitlab-ci.yml" - ".gitlab/ci/build-images.gitlab-ci.yml" - ".gitlab/ci/review.gitlab-ci.yml" + - ".gitlab/ci/review-apps/**/*" - "scripts/review_apps/base-config.yaml" - "scripts/review_apps/review-apps.sh" - "scripts/trigger-build" @@ -150,13 +159,6 @@ - ".markdownlint.yml" - "scripts/lint-doc.sh" -.docs-deprecations-patterns: &docs-deprecations-patterns - - "doc/deprecations/index.md" - - "data/deprecations/*.yml" - - "data/deprecations/templates/_deprecation_template.md.erb" - - "lib/tasks/gitlab/docs/compile_deprecations.rake" - - "tooling/deprecations/docs.rb" - .bundler-patterns: &bundler-patterns - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' @@ -368,13 +370,16 @@ - "danger/**/*" - "tooling/danger/**/*" +.core-backend-patterns: &core-backend-patterns + - "{,jh/}Gemfile{,.lock}" + - "{,ee/,jh/}config/**/*.rb" + .core-frontend-patterns: &core-frontend-patterns - "{package.json,yarn.lock}" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - "config/helpers/**/*.js" - "vendor/assets/javascripts/**/*" - - "{,ee/,jh/}app/assets/**/*.graphql" ################ # Shared rules # @@ -383,11 +388,11 @@ rules: - <<: *if-default-branch-schedule-2-hourly - <<: *if-security-schedule - - <<: *if-merge-request-title-update-caches + - <<: *if-merge-request-labels-update-caches .shared:rules:update-gitaly-binaries-cache: rules: - - <<: *if-merge-request-title-update-caches + - <<: *if-merge-request-labels-update-caches - changes: *gitaly-patterns ###################### @@ -471,12 +476,6 @@ changes: *docs-patterns when: on_success -.docs:rules:deprecations: - rules: - - <<: *if-default-refs - changes: *docs-deprecations-patterns - when: on_success - ################## # GraphQL rules # ################## @@ -502,35 +501,58 @@ .frontend:rules:compile-test-assets: rules: - changes: *code-backstage-qa-patterns - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec .frontend:rules:compile-test-assets-as-if-foss: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-as-if-foss + - <<: *if-merge-request-labels-run-all-rspec + - changes: *code-backstage-qa-patterns + - changes: *startup-css-patterns + +.frontend:rules:compile-test-assets-as-if-jh: + rules: + - <<: *if-not-ee + when: never + - <<: *if-jh + when: never + - <<: *if-merge-request-labels-as-if-jh + - <<: *if-merge-request-labels-run-all-rspec - changes: *code-backstage-qa-patterns - - <<: *if-merge-request-title-run-all-rspec + - changes: *startup-css-patterns .frontend:rules:default-frontend-jobs: rules: - <<: *if-default-refs changes: *code-backstage-patterns -.frontend:rules:default-frontend-jobs-ee: +.frontend:rules:default-frontend-jobs-as-if-foss: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs + - <<: *if-jh + when: never + - <<: *if-security-merge-request changes: *code-backstage-patterns + - <<: *if-merge-request-labels-as-if-foss + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *startup-css-patterns + - <<: *if-merge-request + changes: *ci-patterns -.frontend:rules:default-frontend-jobs-as-if-foss: +.frontend:rules:default-frontend-jobs-as-if-jh: rules: - <<: *if-not-ee when: never + - <<: *if-jh + when: never - <<: *if-security-merge-request changes: *code-backstage-patterns - - <<: *if-merge-request-title-as-if-foss - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-as-if-jh + - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *startup-css-patterns - <<: *if-merge-request @@ -538,7 +560,7 @@ .frontend:rules:jest: rules: - - <<: *if-merge-request-title-run-all-jest + - <<: *if-merge-request-labels-run-all-jest - <<: *if-default-refs changes: *core-frontend-patterns - <<: *if-merge-request @@ -558,7 +580,7 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-jest + - <<: *if-merge-request-labels-run-all-jest when: never - <<: *if-default-refs changes: *core-frontend-patterns @@ -576,7 +598,10 @@ rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-as-if-foss + - <<: *if-jh + when: never + # We already have `static-analysis as-if-foss` which already runs `lint:eslint:all` if the `pipeline:run-as-if-foss` label is set. + - <<: *if-merge-request-labels-as-if-foss when: never - <<: *if-merge-request changes: *frontend-patterns @@ -644,10 +669,12 @@ rules: - <<: *if-not-ee when: never + - <<: *if-jh + when: never - <<: *if-security-merge-request changes: *code-qa-patterns - - <<: *if-merge-request-title-as-if-foss - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-as-if-foss + - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *ci-patterns @@ -673,12 +700,13 @@ ############### .rails:rules:decomposed-databases: rules: - - <<: *if-merge-request-run-decomposed - allow_failure: true + - <<: *if-merge-request-labels-run-decomposed .rails:rules:ee-and-foss-migration: rules: - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-merge-request @@ -695,7 +723,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -708,7 +739,7 @@ rules: - <<: *if-merge-request changes: *db-patterns - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec .rails:rules:db:gitlabcom-database-testing: rules: @@ -720,7 +751,9 @@ .rails:rules:ee-and-foss-unit: rules: - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -735,7 +768,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -745,7 +781,9 @@ .rails:rules:ee-and-foss-integration: rules: - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -760,7 +798,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -770,7 +811,9 @@ .rails:rules:ee-and-foss-system: rules: - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -785,7 +828,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -795,7 +841,9 @@ .rails:rules:ee-and-foss-fast_spec_helper: rules: - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -810,7 +858,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -821,13 +872,15 @@ .rails:rules:code-backstage-qa: rules: - changes: *code-backstage-qa-patterns - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec .rails:rules:ee-only-migration: rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-merge-request @@ -846,7 +899,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -859,7 +915,9 @@ rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -876,7 +934,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -888,7 +949,9 @@ rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -905,7 +968,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -917,7 +983,9 @@ rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -934,7 +1002,10 @@ when: never - <<: *if-automated-merge-request when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + when: never + - <<: *if-merge-request + changes: *core-backend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -946,12 +1017,14 @@ rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-security-merge-request changes: *db-patterns - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *db-patterns - <<: *if-automated-merge-request changes: *db-patterns @@ -967,12 +1040,15 @@ - <<: *if-automated-merge-request when: never - <<: *if-merge-request + changes: *core-backend-patterns + when: never + - <<: *if-merge-request changes: *ci-patterns when: never - <<: *if-security-merge-request changes: *db-patterns when: never - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *db-patterns when: never @@ -980,7 +1056,9 @@ rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -989,7 +1067,7 @@ when: never - <<: *if-security-merge-request changes: *backend-patterns - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *backend-patterns .rails:rules:as-if-foss-unit:minimal: @@ -1001,18 +1079,23 @@ - <<: *if-automated-merge-request when: never - <<: *if-merge-request + changes: *core-backend-patterns + when: never + - <<: *if-merge-request changes: *ci-patterns when: never - <<: *if-security-merge-request changes: *backend-patterns - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *backend-patterns .rails:rules:as-if-foss-integration: rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -1021,7 +1104,7 @@ when: never - <<: *if-security-merge-request changes: *backend-patterns - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *backend-patterns .rails:rules:as-if-foss-integration:minimal: @@ -1033,18 +1116,23 @@ - <<: *if-automated-merge-request when: never - <<: *if-merge-request + changes: *core-backend-patterns + when: never + - <<: *if-merge-request changes: *ci-patterns when: never - <<: *if-security-merge-request changes: *backend-patterns - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *backend-patterns .rails:rules:as-if-foss-system: rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request + changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - <<: *if-automated-merge-request @@ -1053,7 +1141,7 @@ when: never - <<: *if-security-merge-request changes: *code-backstage-patterns - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *code-backstage-patterns .rails:rules:as-if-foss-system:minimal: @@ -1065,23 +1153,26 @@ - <<: *if-automated-merge-request when: never - <<: *if-merge-request + changes: *core-backend-patterns + when: never + - <<: *if-merge-request changes: *ci-patterns when: never - <<: *if-security-merge-request changes: *code-backstage-patterns - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *code-backstage-patterns .rails:rules:ee-and-foss-db-library-code: rules: - changes: *db-library-patterns - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec .rails:rules:ee-mr-and-default-branch-only: rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *code-backstage-patterns - <<: *if-default-branch-refs @@ -1090,13 +1181,13 @@ .rails:rules:detect-tests: rules: - changes: *code-backstage-patterns - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec .rails:rules:rspec-foss-impact: rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss when: never - <<: *if-security-merge-request changes: *code-backstage-patterns @@ -1105,10 +1196,6 @@ .rails:rules:rspec fail-fast: rules: - - <<: *if-rspec-fail-fast-disabled - when: never - - <<: *if-rspec-fail-fast-skipped - when: never - <<: *if-not-ee when: never - <<: *if-security-merge-request @@ -1118,10 +1205,6 @@ .rails:rules:fail-pipeline-early: rules: - - <<: *if-rspec-fail-fast-disabled - when: never - - <<: *if-rspec-fail-fast-skipped - when: never - <<: *if-not-ee when: never - <<: *if-security-merge-request @@ -1136,7 +1219,7 @@ - <<: *if-not-ee when: never - <<: *if-default-branch-schedule-nightly - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec .rails:rules:rspec-coverage: rules: @@ -1146,7 +1229,7 @@ changes: *code-backstage-patterns when: always - <<: *if-default-branch-schedule-2-hourly - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec when: always .rails:rules:default-branch-schedule-nightly--code-backstage: @@ -1181,7 +1264,7 @@ rules: - <<: *if-not-ee when: never - - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request-labels-as-if-foss changes: *code-backstage-qa-patterns - <<: *if-security-merge-request changes: *code-backstage-qa-patterns @@ -1196,7 +1279,7 @@ rules: - <<: *if-merge-request changes: ["vendor/gems/mail-smtp_pool/**/*"] - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec ################## # Releases rules # @@ -1222,75 +1305,76 @@ when: never - <<: *if-default-refs changes: *code-backstage-patterns - allow_failure: true -.reports:rules:sast: +.reports:rules:brakeman-sast: rules: - - if: '$SAST_DISABLED || $GITLAB_FEATURES !~ /\bsast\b/' + - if: $SAST_DISABLED when: never - - <<: *if-default-refs - changes: *code-backstage-qa-patterns - allow_failure: true + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never + - changes: + - '**/*.rb' + - '**/Gemfile' + +.reports:rules:gosec-sast: + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never + - changes: + - '**/*.go' + +.reports:rules:semgrep-sast: + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - changes: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + - '**/*.c' + - '**/*.go' .reports:rules:secret_detection: rules: - if: '$SECRET_DETECTION_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # The Secret-Detection template already has a `secret_detection_default_branch` job - when: never - changes: *code-backstage-qa-patterns - allow_failure: true .reports:rules:gemnasium-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ || $DS_DEFAULT_ANALYZERS !~ /gemnasium([^-]|$)/' when: never - - <<: *if-default-refs - changes: *dependency-patterns - allow_failure: true + - changes: *dependency-patterns .reports:rules:bundler-audit-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/ || $DS_DEFAULT_ANALYZERS !~ /bundler-audit/' when: never - - <<: *if-default-refs - changes: *bundler-patterns - allow_failure: true + - changes: *bundler-patterns .reports:rules:retire-js-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /retire.js/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /retire.js/ || $DS_DEFAULT_ANALYZERS !~ /retire.js/' when: never - - <<: *if-default-refs - changes: *nodejs-patterns - allow_failure: true + - changes: *nodejs-patterns .reports:rules:gemnasium-python-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ || $DS_DEFAULT_ANALYZERS !~ /gemnasium-python/' when: never - - <<: *if-default-refs - changes: *python-patterns - allow_failure: true - -.reports:rules:dast: - rules: - - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' - when: never - - <<: *if-dot-com-gitlab-org-merge-request - changes: *frontend-patterns - allow_failure: true - - <<: *if-dot-com-gitlab-org-merge-request - changes: *code-qa-patterns - when: manual - allow_failure: true + - changes: *python-patterns .reports:rules:schedule-dast: rules: - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' when: never - - <<: *if-dot-com-ee-nightly-schedule - allow_failure: true + - <<: *if-dot-com-ee-nightly-schedule-child-pipeline .reports:rules:package_hunter-yarn: rules: @@ -1310,16 +1394,14 @@ .reports:rules:license_scanning: rules: - - if: '$LICENSE_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\blicense_scanning\b/' + - if: '$LICENSE_MANAGEMENT_DISABLED || $GITLAB_FEATURES !~ /\blicense_scanning\b/' when: never - - <<: *if-default-refs - changes: *code-backstage-qa-patterns - allow_failure: true + - changes: *code-backstage-qa-patterns ################ # Review rules # ################ -.review:rules:review-build-cng: +.review:rules:review-app-pipeline: rules: - <<: *if-not-ee when: never @@ -1336,6 +1418,22 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-schedule +.review:rules:review-build-cng: + rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-review-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *frontend-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-patterns + allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule-child-pipeline + .review:rules:review-deploy: rules: - <<: *if-not-ee @@ -1351,7 +1449,7 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns allow_failure: true - - <<: *if-dot-com-gitlab-org-schedule + - <<: *if-dot-com-gitlab-org-schedule-child-pipeline allow_failure: true .review:rules:review-performance: @@ -1368,7 +1466,7 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns allow_failure: true - - <<: *if-dot-com-gitlab-org-schedule + - <<: *if-dot-com-gitlab-org-schedule-child-pipeline allow_failure: true .review:rules:review-delete-deployment: @@ -1390,7 +1488,7 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns allow_failure: true - - <<: *if-dot-com-ee-schedule + - <<: *if-dot-com-ee-schedule-child-pipeline allow_failure: true # The rule needs to be duplicated between `on_success` and `on_failure` @@ -1418,9 +1516,9 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns when: on_failure - - <<: *if-dot-com-ee-schedule + - <<: *if-dot-com-ee-schedule-child-pipeline when: on_success - - <<: *if-dot-com-ee-schedule + - <<: *if-dot-com-ee-schedule-child-pipeline when: on_failure .review:rules:review-qa-all: @@ -1434,7 +1532,7 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns allow_failure: true - - <<: *if-dot-com-ee-nightly-schedule + - <<: *if-dot-com-ee-nightly-schedule-child-pipeline allow_failure: true # The rule needs to be duplicated between `on_success` and `on_failure` @@ -1456,10 +1554,10 @@ changes: *qa-patterns when: on_failure allow_failure: true - - <<: *if-dot-com-ee-nightly-schedule + - <<: *if-dot-com-ee-nightly-schedule-child-pipeline when: on_success allow_failure: true - - <<: *if-dot-com-ee-nightly-schedule + - <<: *if-dot-com-ee-nightly-schedule-child-pipeline when: on_failure allow_failure: true @@ -1471,7 +1569,7 @@ changes: *code-qa-patterns when: manual allow_failure: true - - <<: *if-dot-com-gitlab-org-schedule + - <<: *if-dot-com-gitlab-org-schedule-child-pipeline allow_failure: true .review:rules:review-stop: @@ -1534,6 +1632,17 @@ changes: *code-backstage-patterns when: on_success +.setup:rules:add-jh-folder: + rules: + - <<: *if-not-ee + when: never + - <<: *if-jh + when: never + - <<: *if-merge-request-labels-as-if-jh + - <<: *if-merge-request-labels-run-all-rspec + - changes: *code-backstage-qa-patterns + - changes: *startup-css-patterns + ####################### # Test metadata rules # ####################### @@ -1541,7 +1650,7 @@ rules: - changes: *code-backstage-patterns when: on_success - - <<: *if-merge-request-title-run-all-rspec + - <<: *if-merge-request-labels-run-all-rspec .test-metadata:rules:update-tests-metadata: rules: diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 60a1ad54cff..eb7a5afad3d 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -101,3 +101,19 @@ detect-tests as-if-foss: MATCHED_TESTS_FILE: tmp/matching_foss_tests.txt before_script: - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' + +add-jh-folder: + extends: .setup:rules:add-jh-folder + image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge + stage: prepare + before_script: + - apk add --no-cache --update curl bash + script: + - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/main-jh/gitlab-main-jh.tar.gz?path=jh" + - tar -xf "jh-folder.tar.gz" + - mv gitlab-main-jh-jh/jh/ ./ + - ls -l jh/ + artifacts: + expire_in: 2d + paths: + - jh/ diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml index 1394085b6e4..85df68e9030 100644 --- a/.gitlab/ci/static-analysis.gitlab-ci.yml +++ b/.gitlab/ci/static-analysis.gitlab-ci.yml @@ -35,6 +35,17 @@ static-analysis: paths: - tmp/feature_flags/ +static-analysis-with-database: + extends: + - .static-analysis-base + - .static-analysis:rules:ee-and-foss + - .use-pg12 + stage: test + script: + - bundle exec rake lint:static_verification_with_database + variables: + SETUP_DB: "true" + static-analysis as-if-foss: extends: - static-analysis diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index ac719977975..2d96fb6d4b0 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -29,8 +29,7 @@ update-tests-metadata: - retrieve-tests-metadata - setup-test-env - rspec migration pg12 - - rspec frontend_fixture - - rspec-ee frontend_fixture + - rspec-all frontend_fixture - rspec unit pg12 - rspec integration pg12 - rspec system pg12 |