diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
| commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
| tree | c42be41678c2586d49a75cabce89322082698334 /.gitlab | |
| parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) | |
| download | gitlab-ce-e8d2c2579383897a1dd7f9debd359abe8ae8373d.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to '.gitlab')
22 files changed, 749 insertions, 308 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 361d1124a78..c7da1c5bebc 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -18,6 +18,7 @@ /doc/administration/monitoring/ @ngaskill /doc/administration/operations/ @axil @eread @marcia /doc/administration/packages/ @ngaskill +/doc/administration/pages/ @axil @kpaizee /doc/administration/postgresql/ @marcia /doc/administration/raketasks/ @axil @eread /doc/administration/redis/ @axil @@ -213,17 +214,20 @@ Dangerfile @gl-quality/eng-prod # Secure & Threat Management ownership delineation # https://about.gitlab.com/handbook/engineering/development/threat-management/delineate-secure-threat-management.html#technical-boundaries [Threat Insights] +/app/finders/security/ @gitlab-org/secure/threat-insights-backend-team /app/models/vulnerability.rb @gitlab-org/secure/threat-insights-backend-team /ee/app/finders/security/ @gitlab-org/secure/threat-insights-backend-team /ee/app/models/security/ @gitlab-org/secure/threat-insights-backend-team /ee/app/models/vulnerabilities/ @gitlab-org/secure/threat-insights-backend-team /ee/app/policies/vulnerabilities/ @gitlab-org/secure/threat-insights-backend-team /ee/app/policies/vulnerability*.rb @gitlab-org/secure/threat-insights-backend-team +/ee/app/presenters/projects/security/ @gitlab-org/secure/threat-insights-backend-team /ee/lib/api/vulnerabilit*.rb @gitlab-org/secure/threat-insights-backend-team /ee/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb @gitlab-org/secure/threat-insights-backend-team /ee/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb @gitlab-org/secure/threat-insights-backend-team /ee/spec/policies/vulnerabilities/ @gitlab-org/secure/threat-insights-backend-team /ee/spec/policies/vulnerability*.rb @gitlab-org/secure/threat-insights-backend-team +/ee/spec/presenters/projects/security/ @gitlab-org/secure/threat-insights-backend-team [Secure] /ee/lib/gitlab/ci/parsers/license_compliance/ @gitlab-org/secure/composition-analysis-be @@ -232,6 +236,7 @@ Dangerfile @gl-quality/eng-prod /ee/lib/gitlab/ci/reports/dependency_list/ @gitlab-org/secure/composition-analysis-be /ee/lib/gitlab/ci/reports/license_scanning/ @gitlab-org/secure/composition-analysis-be /ee/lib/gitlab/ci/reports/security/ @gitlab-org/secure/composition-analysis-be @gitlab-org/secure/dynamic-analysis-be @gitlab-org/secure/static-analysis-be @gitlab-org/secure/fuzzing-be +/ee/app/services/ci/run_dast_scan_service.rb @gitlab-org/secure/dynamic-analysis-be [Container Security] /ee/app/views/projects/threat_monitoring/** @gitlab-org/protect/container-security-frontend diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index ed1f71e27bb..853f92ed98d 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -1,43 +1,48 @@ -# This image is used by the `review-qa-*` jobs. The image name is also passed to the downstream `omnibus-gitlab-mirror` pipeline -# triggered by `package-and-qa` so that it doesn't have to rebuild it a second time. The downstream `omnibus-gitlab-mirror` pipeline -# itself passes the image name to the `gitlab-qa-mirror` pipeline so that it can use it instead of inferring an end-to-end image -# from the GitLab image built by the downstream `omnibus-gitlab-mirror` pipeline. +.base-image-build: + extends: .use-kaniko + script: + # With .git/hooks/post-checkout in place, Git tries to pull LFS objects, but the image doesn't have Git LFS, and we actually don't care about it for this specific so we just remove the file. + # Without removing the file, the error is as follows: "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout." + - rm -f .git/hooks/post-checkout + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + echo "Checking out \$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ($CI_MERGE_REQUEST_SOURCE_BRANCH_SHA) instead of \$CI_COMMIT_SHA (merge result commit $CI_COMMIT_SHA) so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync."; + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + else + echo "Building the image from \$CI_COMMIT_SHA ($CI_COMMIT_SHA) for this non-merge result pipeline."; + fi; + - echo "See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results for more details."; + retry: 2 + +# This image is used by: +# - The `review-qa-*` jobs +# - The downstream `omnibus-gitlab-mirror` pipeline triggered by `package-and-qa` so that it doesn't have to rebuild it again. +# The downstream `omnibus-gitlab-mirror` pipeline itself passes the image name to the `gitlab-qa-mirror` pipeline so that +# it can use it instead of inferring an end-to-end imag from the GitLab image built by the downstream `omnibus-gitlab-mirror` pipeline. # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#testing-code-in-merge-requests for more details. build-qa-image: extends: - - .use-kaniko + - .base-image-build - .build-images:rules:build-qa-image stage: build-images needs: [] variables: QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" script: - # With .git/hooks/post-checkout in place, Git tries to pull LFS objects, but the image doesn't have Git LFS, and we actually don't care about it for this specific so we just remove the file. - # Without removing the file, the error is as follows: "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout." - - rm .git/hooks/post-checkout - # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. - # This falls back to $CI_COMMIT_SHA (the default checked out commit) for the non-merged result pipelines. - # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results. - - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then - git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; - fi + - !reference [.base-image-build, script] - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true - retry: 2 # This image is used by: # - The `CNG` pipelines (via the `review-build-cng` job): https://gitlab.com/gitlab-org/build/CNG/-/blob/cfc67136d711e1c8c409bf8e57427a644393da2f/.gitlab-ci.yml#L335 # - The `omnibus-gitlab` pipelines (via the `package-and-qa` job): https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/dfd1ad475868fc84e91ab7b5706aa03e46dc3a86/.gitlab-ci.yml#L130 build-assets-image: extends: - - .use-kaniko + - .base-image-build - .build-images:rules:build-assets-image stage: build-images needs: ["compile-production-assets"] - variables: - GIT_DEPTH: "1" script: + - !reference [.base-image-build, script] # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines # https://gitlab.com/gitlab-org/gitlab/issues/208389 - run_timed_command "scripts/build_assets_image" - retry: 2 diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index a4317c72413..60025c244f1 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -53,7 +53,7 @@ docs-lint markdown: docs-lint links: extends: - .docs:rules:docs-lint - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.13-ruby-2.7.2 + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.13-ruby-2.7.2-d2b92621 stage: test needs: [] script: diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index dfd595c2696..3aa8ab1591a 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -2,6 +2,10 @@ - source scripts/utils.sh - run_timed_command "retry yarn install --frozen-lockfile" +.storybook-yarn-install: &storybook-yarn-install + - source scripts/utils.sh + - run_timed_command "retry yarn run storybook:install --frozen-lockfile" + .compile-assets-base: extends: - .default-retry @@ -35,6 +39,15 @@ compile-production-assets: - public/assets/ - webpack-report/ when: always + before_script: + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + echo "Checking out \$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ($CI_MERGE_REQUEST_SOURCE_BRANCH_SHA) instead of \$CI_COMMIT_SHA (merge result commit $CI_COMMIT_SHA) so that GitLab assets image tag actually reflect the commit for which assets were compiled."; + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + else + echo "Building the image from \$CI_COMMIT_SHA ($CI_COMMIT_SHA) for this non-merge result pipeline."; + fi; + - echo "See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results for more details."; + - !reference [.default-before_script, before_script] after_script: - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here @@ -80,6 +93,15 @@ update-yarn-cache: script: - *yarn-install +update-storybook-yarn-cache: + extends: + - .default-retry + - .storybook-yarn-cache-push + - .shared:rules:update-cache + stage: prepare + script: + - *storybook-yarn-install + .frontend-fixtures-base: extends: - .default-retry @@ -94,7 +116,7 @@ update-yarn-cache: - run_timed_command "gem install knapsack --no-document" - run_timed_command "scripts/gitaly-test-spawn" - source ./scripts/rspec_helpers.sh - - rspec_paralellized_job "--tag frontend_fixture" + - rspec_paralellized_job artifacts: name: frontend-fixtures expire_in: 31d @@ -344,3 +366,29 @@ startup-css-check as-if-foss: needs: - job: "compile-test-assets as-if-foss" - job: "rspec frontend_fixture as-if-foss" + +.compile-storybook-base: + extends: + - .frontend-test-base + - .storybook-yarn-cache + script: + - *yarn-install # storybook depends on the global webpack config, so we must install global deps. + - *storybook-yarn-install + - yarn run storybook:build + +compile-storybook: + extends: + - .compile-storybook-base + - .frontend:rules:default-frontend-jobs + artifacts: + name: storybook + expire_in: 31d + when: always + paths: + - storybook/public + +compile-storybook as-if-foss: + extends: + - .compile-storybook-base + - .as-if-foss + - .frontend:rules:default-frontend-jobs-as-if-foss diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index ba49ddfce9d..860397b9437 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -75,7 +75,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .assets-cache: &assets-cache - key: "assets-${NODE_ENV}-v1" + key: "assets-${NODE_ENV}-v2" paths: - assets-hash.txt - public/assets/webpack/ @@ -88,6 +88,16 @@ <<: *assets-cache policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. +.storybook-node-modules-cache: &storybook-node-modules-cache + key: "storybook-node-modules-${NODE_ENV}-v1" + paths: + - storybook/node_modules/ + policy: pull + +.storybook-node-modules-cache-push: &storybook-node-modules-cache-push + <<: *storybook-node-modules-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + .rubocop-cache: &rubocop-cache key: "rubocop-v1" paths: @@ -181,6 +191,14 @@ - *node-modules-cache-push - *assets-cache-push +.storybook-yarn-cache: + cache: + - *storybook-node-modules-cache + +.storybook-yarn-cache-push: + cache: + - *storybook-node-modules-cache-push + .use-pg11: image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: @@ -199,6 +217,15 @@ variables: POSTGRES_HOST_AUTH_METHOD: trust +.use-pg13: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + services: + - name: postgres:13 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:5.0-alpine + variables: + POSTGRES_HOST_AUTH_METHOD: trust + .use-pg11-ee: image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: @@ -221,6 +248,17 @@ variables: POSTGRES_HOST_AUTH_METHOD: trust +.use-pg13-ee: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + services: + - name: postgres:13 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:5.0-alpine + - name: elasticsearch:7.11.1 + command: ["elasticsearch", "-E", "discovery.type=single-node"] + variables: + POSTGRES_HOST_AUTH_METHOD: trust + .use-kaniko: image: name: registry.gitlab.com/gitlab-org/gitlab-build-images:kaniko diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index b6d4b0ef11d..e75b0db9071 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -1,3 +1,7 @@ +.compress-public: &compress-public + - find public -type f -regex '.*\.\(htm\|html\|txt\|text\|js\|json\|css\|svg\|xml\)$' -exec gzip -f -k {} \; + - find public -type f -regex '.*\.\(htm\|html\|txt\|text\|js\|json\|css\|svg\|xml\)$' -exec brotli -f -k {} \; + pages: extends: - .default-retry @@ -8,14 +12,18 @@ pages: - coverage-frontend - karma - compile-production-assets + - compile-storybook + before_script: + - apt-get update && apt-get -y install brotli gzip script: - mv public/ .public/ - mkdir public/ - mv coverage/ public/coverage-ruby/ || true - mv coverage-frontend/ public/coverage-frontend/ || true - mv coverage-javascript/ public/coverage-javascript/ || true + - mv storybook/public public/storybook || true - cp .public/assets/application-*.css public/application.css || true - - cp .public/assets/application-*.css.gz public/application.css.gz || true + - *compress-public artifacts: paths: - public diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index b74c8ebf575..0dfc353820f 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -47,7 +47,6 @@ - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ - - tmp/feature_flags/ - log/*.log reports: junit: junit_rspec.xml @@ -75,6 +74,11 @@ - .use-pg12 needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] +.rspec-base-pg13: + extends: + - .rspec-base + - .use-pg13 + .rspec-ee-base-pg11: extends: - .rspec-base @@ -85,6 +89,11 @@ - .rspec-base - .use-pg12-ee +.rspec-ee-base-pg13: + extends: + - .rspec-base + - .use-pg13-ee + .rspec-ee-base-geo: extends: .rspec-base script: @@ -101,6 +110,11 @@ - .rspec-ee-base-geo - .use-pg12-ee +.rspec-ee-base-geo-pg13: + extends: + - .rspec-ee-base-geo + - .use-pg13-ee + .db-job-base: extends: - .rails-job-base @@ -114,7 +128,7 @@ ############################ # rspec job parallel configs .rspec-migration-parallel: - parallel: 7 + parallel: 9 .rspec-ee-migration-parallel: parallel: 3 @@ -123,7 +137,7 @@ parallel: 20 .rspec-ee-unit-parallel: - parallel: 10 + parallel: 12 .rspec-ee-unit-geo-parallel: parallel: 2 @@ -149,13 +163,16 @@ setup-test-env: - .rails-job-base - .setup-test-env-cache - .rails:rules:code-backstage-qa - - .use-pg12 stage: prepare variables: GITLAB_TEST_EAGER_LOAD: "0" + SETUP_DB: "false" script: - - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'" + - run_timed_command "scripts/setup-test-env" + - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-build[collapsed=true]\r\e[0KCompiling Gitaly binaries" - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here + - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-build\r\e[0K" + artifacts: expire_in: 7d paths: @@ -237,6 +254,11 @@ static-analysis: script: - run_timed_command "retry yarn install --frozen-lockfile" - scripts/static-analysis + artifacts: + expire_in: 31d + when: always + paths: + - tmp/feature_flags/ static-analysis as-if-foss: extends: @@ -244,6 +266,22 @@ static-analysis as-if-foss: - .static-analysis:rules:as-if-foss - .as-if-foss +zeitwerk-check: + extends: + - .rails-cache + - .default-before_script + - .rails:rules:ee-and-foss-unit + variables: + BUNDLE_WITHOUT: "" + SETUP_DB: "false" + needs: [] + stage: test + script: + - sed -i -e "s/config\.autoloader = :classic/config\.autoloader = :zeitwerk/" config/application.rb + - RAILS_ENV=test bundle exec rake zeitwerk:check + - RAILS_ENV=development bundle exec rake zeitwerk:check + - RAILS_ENV=production bundle exec rake zeitwerk:check + rspec migration pg12: extends: - .rspec-base-pg12 @@ -355,11 +393,10 @@ db:migrate-from-previous-major-version: - sed -i -e "s/gem 'mimemagic', '~> 0.3.2'/gem 'ruby-magic', '~> 0.4.0'/" Gemfile - run_timed_command "gem install bundler:1.17.3" - run_timed_command "bundle update google-protobuf nokogiri grpc mimemagic bootsnap" - - run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}" - - cp config/gitlab.yml.example config/gitlab.yml + - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh - run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu" - git checkout -f $CI_COMMIT_SHA - - run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}" + - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh - run_timed_command "bundle exec rake db:migrate" db:rollback: @@ -487,23 +524,7 @@ rspec:feature-flags: - .coverage-base - .rails:rules:rspec-feature-flags stage: post-test - # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) - # so we use `dependencies` here. - dependencies: - - setup-test-env - - rspec migration pg12 - - rspec unit pg12 - - rspec integration pg12 - - rspec system pg12 - - rspec-ee migration pg12 - - rspec-ee unit pg12 - - rspec-ee integration pg12 - - rspec-ee system pg12 - - rspec-ee unit pg12 geo - - rspec-ee integration pg12 geo - - rspec-ee system pg12 geo - - memory-static - - memory-on-boot + needs: ["static-analysis"] script: - !reference [.minimal-bundle-install, script] - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then @@ -664,6 +685,8 @@ db:rollback geo: ########################################## # EE/FOSS: default branch nightly scheduled jobs # + +# PG11 rspec migration pg11: extends: - .rspec-base-pg11 @@ -688,11 +711,40 @@ rspec system pg11: - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-system-parallel + +# PG13 +rspec migration pg13: + extends: + - .rspec-base-pg13 + - .rspec-base-migration + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-migration-parallel + +rspec unit pg13: + extends: + - .rspec-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-unit-parallel + +rspec integration pg13: + extends: + - .rspec-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-integration-parallel + +rspec system pg13: + extends: + - .rspec-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-system-parallel + # EE/FOSS: default branch nightly scheduled jobs # ########################################## ##################################### # EE: default branch nightly scheduled jobs # + +# PG11 rspec-ee migration pg11: extends: - .rspec-ee-base-pg11 @@ -733,6 +785,48 @@ rspec-ee system pg11 geo: extends: - .rspec-ee-base-geo-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + +# PG13 +rspec-ee migration pg13: + extends: + - .rspec-ee-base-pg13 + - .rspec-base-migration + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-migration-parallel + +rspec-ee unit pg13: + extends: + - .rspec-ee-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-unit-parallel + +rspec-ee integration pg13: + extends: + - .rspec-ee-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-integration-parallel + +rspec-ee system pg13: + extends: + - .rspec-ee-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-system-parallel + +rspec-ee unit pg13 geo: + extends: + - .rspec-ee-base-geo-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-unit-geo-parallel + +rspec-ee integration pg13 geo: + extends: + - .rspec-ee-base-geo-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + +rspec-ee system pg13 geo: + extends: + - .rspec-ee-base-geo-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only # EE: default branch nightly scheduled jobs # ##################################### diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 4cc03fdb1a4..690a971927c 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -74,33 +74,35 @@ gemnasium-dependency_scanning: - apk add jq # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390 - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:gemnasium-dependency_scanning", rules] bundler-audit-dependency_scanning: - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:bundler-audit-dependency_scanning", rules] retire-js-dependency_scanning: - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:retire-js-dependency_scanning", rules] gemnasium-python-dependency_scanning: - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:gemnasium-python-dependency_scanning", rules] # Analyze dependencies for malicious behavior # See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter -package_hunter: +.package_hunter-base: extends: - .default-retry - - .reports:rules:package_hunter stage: test image: - name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:latest + name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:1.1.0 entrypoint: [""] + variables: + DEBUG: '*' + HTR_user: '$PACKAGE_HUNTER_USER' + HTR_pass: '$PACKAGE_HUNTER_PASS' needs: [] allow_failure: true - script: + before_script: - rm -r spec locale .git app/assets/images doc/ - cd .. && tar -I "gzip --best" -cf gitlab.tgz gitlab/ - - DEBUG=* HTR_user=$PACKAGE_HUNTER_USER HTR_pass=$PACKAGE_HUNTER_PASS node /usr/src/app/cli.js analyze --format gitlab gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json artifacts: paths: - gl-dependency-scanning-report.json @@ -108,6 +110,20 @@ package_hunter: dependency_scanning: gl-dependency-scanning-report.json expire_in: 1 week +package_hunter-yarn: + extends: + - .package_hunter-base + - .reports:rules:package_hunter-yarn + script: + - node /usr/src/app/cli.js analyze --format gitlab --manager yarn gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json + +package_hunter-bundler: + extends: + - .package_hunter-base + - .reports:rules:package_hunter-bundler + script: + - node /usr/src/app/cli.js analyze --format gitlab --manager bundler gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json + license_scanning: extends: .default-retry needs: [] diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index cabeb0cc1ac..38d59af5aed 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -2,7 +2,7 @@ review-cleanup: extends: - .default-retry - .review:rules:review-cleanup - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3.5-kubectl1.17 stage: prepare environment: name: review/auto-cleanup @@ -25,7 +25,7 @@ review-build-cng: extends: - .default-retry - .review:rules:review-build-cng - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 stage: review-prepare before_script: - source ./scripts/utils.sh @@ -54,7 +54,6 @@ review-deploy: extends: - .review-workflow-base - .review:rules:review-deploy - retry: 2 stage: review needs: ["review-build-cng"] resource_group: "review/${CI_COMMIT_REF_NAME}" @@ -66,8 +65,6 @@ review-deploy: - *base-before_script script: - check_kube_domain - - "ensure_namespace ${KUBE_NAMESPACE}" - - install_external_dns - download_chart - date - deploy || (display_deployment_debug && exit 1) @@ -97,13 +94,13 @@ review-deploy: before_script: - *base-before_script -review-stop-failed-deployment: +review-delete-deployment: extends: - .review-stop-base - - .review:rules:review-stop-failed-deployment + - .review:rules:review-delete-deployment stage: prepare script: - - delete_failed_release + - delete_release review-stop: extends: @@ -111,18 +108,19 @@ review-stop: - .review:rules:review-stop stage: post-qa script: - - delete_release + - delete_k8s_release_namespace .review-qa-base: extends: - .use-docker-in-docker - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7 + image: + name: ${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG} + entrypoint: [""] stage: qa - needs: ["review-deploy"] + needs: ["build-qa-image", "review-deploy"] variables: - QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" - QA_CAN_TEST_GIT_PROTOCOL_V2: "false" QA_DEBUG: "true" + QA_CAN_TEST_GIT_PROTOCOL_V2: "false" QA_GENERATE_ALLURE_REPORT: "true" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" @@ -132,21 +130,22 @@ review-stop: EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" SIGNUP_DISABLED: "true" before_script: - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" + # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + fi - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - - echo "${QA_IMAGE}" - - *base-before_script - - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} + - cd qa artifacts: paths: - - ./qa/gitlab-qa-run-* + - qa/tmp expire_in: 7 days when: always .allure-report-base: image: - name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.2 + name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.4 entrypoint: [""] stage: post-qa variables: @@ -157,7 +156,7 @@ review-stop: script: - | allure-report-publisher upload gcs \ - --results-glob="qa/gitlab-qa-run-*/**/allure-results/*" \ + --results-glob="qa/tmp/allure-results/*" \ --bucket="gitlab-qa-allure-reports" \ --prefix="$ALLURE_REPORT_PATH_PREFIX/$CI_COMMIT_REF_SLUG" \ --update-pr="comment" \ @@ -171,7 +170,7 @@ review-qa-smoke: - .review:rules:review-qa-smoke retry: 1 # This is confusing but this means "2 runs at max". script: - - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" + - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}" review-qa-all: extends: @@ -181,7 +180,14 @@ review-qa-all: script: - export KNAPSACK_REPORT_PATH=knapsack/master_report.json - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb - - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation + - | + bin/test Test::Instance::All "${CI_ENVIRONMENT_URL}" \ + -- \ + --color --format documentation \ + --format RspecJunitFormatter --out tmp/rspec.xml + artifacts: + reports: + junit: qa/tmp/rspec.xml review-performance: extends: @@ -209,32 +215,6 @@ review-performance: performance: performance.json expire_in: 31d -parallel-spec-reports: - extends: - - .review:rules:review-qa-all - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine - stage: post-qa - needs: ["review-qa-all"] - variables: - NEW_PARALLEL_SPECS_REPORT: qa/report-new.html - BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" - script: - - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* - - gem install nokogiri --no-document - - cd qa/gitlab-qa-run-*/gitlab-* - - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_) - - cd - - - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' - - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm - artifacts: - when: always - paths: - - qa/report-new.html - - qa/gitlab-qa-run-* - reports: - junit: qa/gitlab-qa-run-*/**/rspec-*.xml - expire_in: 31d - allure-report-qa-smoke: extends: - .allure-report-base @@ -267,10 +247,14 @@ danger-review: script: - > if [ -z "$DANGER_GITLAB_API_TOKEN" ]; then - # Force danger to skip CI source GitLab and fallback to "local only git repo". - unset GITLAB_CI - # We need to base SHA to help danger determine the base commit for this shallow clone. - run_timed_command "bundle exec danger dry_run --fail-on-errors=true --verbose --base='$CI_MERGE_REQUEST_DIFF_BASE_SHA'" + run_timed_command danger_as_local else run_timed_command "bundle exec danger --fail-on-errors=true --verbose" fi + +danger-review-local: + extends: + - danger-review + - .review:rules:danger-local + script: + - run_timed_command danger_as_local diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 8dd97c1fe69..c97ef4fc868 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -131,6 +131,30 @@ - ".markdownlint.yml" - "scripts/lint-doc.sh" +.bundler-patterns: &bundler-patterns + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + +.nodejs-patterns: &nodejs-patterns + - '{package.json,*/package.json,*/*/package.json}' + +.python-patterns: &python-patterns + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + +.dependency-patterns: &dependency-patterns + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' + .frontend-dependency-patterns: &frontend-dependency-patterns - "{package.json,yarn.lock}" - "config/webpack.config.js" @@ -138,6 +162,7 @@ .frontend-build-patterns: &frontend-build-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "config/webpack.config.js" - "config/**/*.js" @@ -146,6 +171,7 @@ .frontend-patterns: &frontend-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -163,7 +189,7 @@ - "config.ru" # List explicitly all the app/ dirs that are backend (i.e. all except app/assets). - "{,ee/,jh/}{app/channels,app/controllers,app/finders,app/graphql,app/helpers,app/mailers,app/models,app/policies,app/presenters,app/serializers,app/services,app/uploaders,app/validators,app/views,app/workers}/**/*" - - "{,ee/,jh/}{bin,cable,config,db,lib}/**/*" + - "{,ee/,jh/}{bin,cable,config,db,generator_templates,lib}/**/*" - "{,ee/,jh/}spec/**/*.rb" # CI changes - ".gitlab-ci.yml" @@ -198,6 +224,7 @@ .code-patterns: &code-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -213,7 +240,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" @@ -223,6 +250,7 @@ .code-backstage-patterns: &code-backstage-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -238,7 +266,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" # Backstage changes @@ -251,6 +279,7 @@ .code-qa-patterns: &code-qa-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -266,7 +295,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" # QA changes @@ -275,6 +304,7 @@ .code-backstage-qa-patterns: &code-backstage-qa-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -290,7 +320,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" # Backstage changes @@ -304,6 +334,12 @@ - ".dockerignore" - "qa/**/*" +.code-backstage-danger-patterns: &code-backstage-danger-patterns + # Backstage changes + - "Dangerfile" + - "danger/**/*" + - "tooling/danger/**/*" + ################ # Shared rules # ################ @@ -932,14 +968,6 @@ - <<: *if-merge-request-title-run-all-rspec when: always -.rails:rules:rspec-feature-flags: - rules: - - <<: *if-not-ee - when: never - - <<: *if-default-branch-schedule-2-hourly - allow_failure: true - - <<: *if-merge-request-title-run-all-rspec - .rails:rules:default-branch-schedule-nightly--code-backstage: rules: - <<: *if-default-branch-schedule-nightly @@ -954,6 +982,12 @@ - <<: *if-merge-request changes: [".gitlab/ci/rails.gitlab-ci.yml"] +.rails:rules:rspec-feature-flags: + rules: + - <<: *if-not-ee + when: never + - changes: *code-backstage-patterns + ######################### # Static analysis rules # ######################### @@ -1005,7 +1039,6 @@ rules: - if: '$CODE_QUALITY_DISABLED' when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-patterns allow_failure: true @@ -1014,7 +1047,6 @@ rules: - if: '$SAST_DISABLED || $GITLAB_FEATURES !~ /\bsast\b/' when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns allow_failure: true @@ -1025,17 +1057,39 @@ when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # The Secret-Detection template already has a `secret_detection_default_branch` job when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - changes: *code-backstage-qa-patterns allow_failure: true -.reports:rules:dependency_scanning: +.reports:rules:gemnasium-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/' when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs - changes: *code-backstage-qa-patterns + changes: *dependency-patterns + allow_failure: true + +.reports:rules:bundler-audit-dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/' + when: never + - <<: *if-default-refs + changes: *bundler-patterns + allow_failure: true + +.reports:rules:retire-js-dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /retire.js/' + when: never + - <<: *if-default-refs + changes: *nodejs-patterns + allow_failure: true + +.reports:rules:gemnasium-python-dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/' + when: never + - <<: *if-default-refs + changes: *python-patterns allow_failure: true .reports:rules:dast: @@ -1057,12 +1111,22 @@ - <<: *if-default-branch-schedule-nightly allow_failure: true -.reports:rules:package_hunter: +.reports:rules:package_hunter-yarn: rules: + - if: "$PACKAGE_HUNTER_USER == null || $PACKAGE_HUNTER_USER == ''" + when: never - <<: *if-default-branch-schedule-2-hourly - <<: *if-merge-request changes: ["yarn.lock"] +.reports:rules:package_hunter-bundler: + rules: + - if: "$PACKAGE_HUNTER_USER == null || $PACKAGE_HUNTER_USER == ''" + when: never + - <<: *if-default-branch-schedule-2-hourly + - <<: *if-merge-request + changes: ["Gemfile.lock"] + .reports:rules:license_scanning: rules: - if: '$LICENSE_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\blicense_scanning\b/' @@ -1126,7 +1190,7 @@ - <<: *if-dot-com-gitlab-org-schedule allow_failure: true -.review:rules:review-stop-failed-deployment: +.review:rules:review-delete-deployment: rules: - <<: *if-not-ee when: never @@ -1146,19 +1210,31 @@ changes: *code-qa-patterns allow_failure: true +# The rule needs to be duplicated between `on_success` and `on_failure` +# because the jobs `needs` the previous job to complete. +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 .review:rules:review-qa-smoke-report: rules: - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-review-patterns + when: on_failure - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *frontend-patterns + when: on_failure - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: on_failure .review:rules:review-qa-all: rules: @@ -1172,6 +1248,9 @@ changes: *qa-patterns allow_failure: true +# The rule needs to be duplicated between `on_success` and `on_failure` +# because the jobs `needs` the previous job to complete. +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 .review:rules:review-qa-all-report: rules: - <<: *if-not-ee @@ -1183,7 +1262,11 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns allow_failure: true - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns + allow_failure: true + when: on_failure .review:rules:review-cleanup: rules: @@ -1211,6 +1294,11 @@ rules: - if: '$CI_MERGE_REQUEST_IID' +.review:rules:danger-local: + rules: + - if: '$CI_MERGE_REQUEST_IID' + changes: *code-backstage-danger-patterns + ############### # Setup rules # ############### diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 27b68115edc..f2d5d872d64 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -52,7 +52,7 @@ no_ee_check: verify-tests-yml: extends: - .setup:rules:verify-tests-yml - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 stage: test needs: [] script: diff --git a/.gitlab/issue_templates/Deprecations.md b/.gitlab/issue_templates/Deprecations.md new file mode 100644 index 00000000000..ff51699c6be --- /dev/null +++ b/.gitlab/issue_templates/Deprecations.md @@ -0,0 +1,50 @@ +<!-- Use this template as a starting point for deprecations. --> + +### Deprecation Summary + +<!-- +This should contain a brief description of the feature or functionality that is deprecated. The description should clearly state the potential impact of the deprecation to end users. + +It is recommended that you link to the documentation. + +The description of the deprecation should state what actions the user should take to rectify the behavior. If the deprecation is scheduled for an upcoming release, the content should remain in the deprecations documentation page until it has been completed. For example, if a deprecation is announced in 14.9 and scheduled to be completed in 15.0, the same content would be included in the documentation for 14.9, 14.10, and 15.0. +--> + +### Breaking Change + +<!-- Is this a breaking change or not? If so, please add instructions for how users can update their workflow. --> +### Affected Topology + +<!-- +Who is affected by this deprecation, Self-managed users, SaaS users, or both? This is especially important when nearing the annual major release where breaking changes and removals are typically introduced. These changes might be seen on GitLab.com before the official release date. +--> + +### Affected Tier + +<!-- +Which tier is this feature available in? + +* Free +* Premium +* Ultimate +--> + +### Checklist + +- [ ] @mention your stage's stable counterparts on this issue. For example, Customer Support, Customer Success (Technical Account Manager), Product Marketing Manager. + - To see who the stable counterparts are for a product team visit [product categories](https://about.gitlab.com/handbook/product/categories/) + - If there is no stable counterpart listed for Sales/CS please mention `@timtams` + - If there is no stable counterpart listed for Support please @mention `@gitlab-com/support/managers` + - If there is no stable counterpart listed for Marketing please mention `@williamchia` + +- [ ] @mention your GPM so that they are aware of planned deprecations. The goal is to have reviews happen at least two releases before the final removal of the feature or introduction of a breaking change. + +### Deprecation Milestone + +<!-- In which milestone will this deprecation happen? --> + +### Links + +<!-- +Add links to any relevant documentation or code that will provide additional details or clarity regarding the planned change. Also, include a link to the removal issue if relevant. +--> diff --git a/.gitlab/issue_templates/Design Sprint.md b/.gitlab/issue_templates/Design Sprint.md index 787c4469b88..06ef7dc9abc 100644 --- a/.gitlab/issue_templates/Design Sprint.md +++ b/.gitlab/issue_templates/Design Sprint.md @@ -98,6 +98,7 @@ If you enjoy taking notes using post-it notes make sure you have available some - [ ] Finalise participant list - `decider` and `facilitator` - [ ] Create [participation form](https://docs.google.com/forms/d/e/1FAIpQLSc0_BNltvRW8yXXaJd8sIKzgDmrSGqILMfkoCJrAj6sFcsMcg/viewform?usp=sf_link) and send to participants (**deadline**: [date]) - `facilitator` +- [ ] Create a dedicated Slack channel and add participants - `facilitator` - [ ] Promote this issue to an epic - `facilitator` - [ ] Create issues under the epic for the pre-workshop tasks: Expert interviews ([example](https://gitlab.com/groups/gitlab-org/configure/-/epics/3#note_332412524)), Lightning walkthroughs and How might we.. notetaking assignment ([example](https://gitlab.com/gitlab-org/configure/general/-/issues/52)), Voting How might we... notes assignment ([example](https://gitlab.com/gitlab-org/configure/general/-/issues/54)) - `facilitator` - [ ] Create sync meetings in calendar and invite all participants (**deadline**: [date]) - `decider` or `facilitator` diff --git a/.gitlab/issue_templates/Feature Flag Roll Out.md b/.gitlab/issue_templates/Feature Flag Roll Out.md index 424b752d1b4..ec6e5dfd7d4 100644 --- a/.gitlab/issue_templates/Feature Flag Roll Out.md +++ b/.gitlab/issue_templates/Feature Flag Roll Out.md @@ -78,6 +78,7 @@ Are there any other stages or teams involved that need to be kept in the loop? If a different developer will be covering, or an exception is needed, please inform the oncall SRE by using the `@sre-oncall` Slack alias. - [ ] Ensure that documentation has been updated ([More info](https://docs.gitlab.com/ee/development/documentation/feature_flags.html#features-that-became-enabled-by-default)). - [ ] Announce on [the feature issue](ISSUE LINK) an estimated time this will be enabled on GitLab.com. +- [ ] If the feature might impact the user experience, notify `#support_gitlab-com` and your team channel ([more guidance when this is necessary in the dev docs](https://docs.gitlab.com/ee/development/feature_flags/controls.html#communicate-the-change)). - [ ] If the feature flag in code has [an actor](https://docs.gitlab.com/ee/development/feature_flags/#feature-actors), enable it on GitLab.com for [testing groups/projects](#testing-groupsprojectsusers). - [ ] `/chatops run feature set --<actor-type>=<actor> <feature-flag-name> true` - [ ] Verify that the feature works as expected. Posting the QA result in this issue is preferable. @@ -92,8 +93,6 @@ Are there any other stages or teams involved that need to be kept in the loop? - Enable the feature globally on production environment. - [ ] `/chatops run feature set <feature-flag-name> true` - [ ] Announce on [the feature issue](ISSUE LINK) that the feature has been globally enabled. -- [ ] Cross-post chatops slack command to `#support_gitlab-com`. - ([more guidance when this is necessary in the dev docs](https://docs.gitlab.com/ee/development/feature_flags/controls.html#communicate-the-change)) and in your team channel - [ ] Wait for [at least one day for the verification term](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#including-a-feature-behind-feature-flag-in-the-final-release). ### (Optional) Release the feature with the feature flag @@ -105,10 +104,10 @@ To do so, follow these steps: - [ ] Create a merge request with the following changes. Ask for review and merge it. - [ ] Set the `default_enabled` attribute in [the feature flag definition](https://docs.gitlab.com/ee/development/feature_flags/#feature-flag-definition-and-validation) to `true`. - [ ] Create [a changelog entry](https://docs.gitlab.com/ee/development/feature_flags/#changelog). -- [ ] Ensure that the above MR has been deployed to both production and canary. +- [ ] Ensure that the default-enabling MR has been deployed to both production and canary. If the merge request was deployed before [the code cutoff](https://about.gitlab.com/handbook/engineering/releases/#self-managed-releases-1), the feature can be officially announced in a release blog post. - - [ ] `/chatops run auto_deploy status <merge-commit>` + - [ ] `/chatops run auto_deploy status <merge-commit-of-default-enabling-mr>` - [ ] Close [the feature issue](ISSUE LINK) to indicate the feature will be released in the current milestone. - [ ] Set the next milestone to this rollout issue for scheduling [the flag removal](#release-the-feature). - [ ] (Optional) You can create a separate issue for scheduling the steps below to [Release the feature](#release-the-feature). @@ -136,10 +135,10 @@ codebase. - [ ] Remove all references to the feature flag from the codebase. - [ ] Remove the YAML definitions for the feature from the repository. - [ ] Create [a changelog entry](https://docs.gitlab.com/ee/development/feature_flags/#changelog). -- [ ] Ensure that the above MR has been deployed to both production and canary. +- [ ] Ensure that the cleanup MR has been deployed to both production and canary. If the merge request was deployed before [the code cutoff](https://about.gitlab.com/handbook/engineering/releases/#self-managed-releases-1), the feature can be officially announced in a release blog post. - - [ ] `/chatops run auto_deploy status <merge-commit>` + - [ ] `/chatops run auto_deploy status <merge-commit-of-cleanup-mr>` - [ ] Close [the feature issue](ISSUE LINK) to indicate the feature will be released in the current milestone. - [ ] Clean up the feature flag from all environments by running these chatops command in `#production` channel: - [ ] `/chatops run feature delete <feature-flag-name> --dev` diff --git a/.gitlab/issue_templates/Feature Proposal - basic.md b/.gitlab/issue_templates/Feature Proposal - basic.md index 099243c05ca..0c05b7a0165 100644 --- a/.gitlab/issue_templates/Feature Proposal - basic.md +++ b/.gitlab/issue_templates/Feature Proposal - basic.md @@ -5,7 +5,6 @@ <!-- Use this section to explain the feature and how it will work. It can be helpful to add technical details, design proposals, and links to related epics or issues. --> <!-- Consider adding related issues and epics to this issue. You can also reference the Feature Proposal Template (https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20proposal%20-%20detailed.md) for additional details to consider adding to this issue. Additionally, as a data oriented organization, when your feature exits planning breakdown, consider adding the `What does success look like, and how can we measure that?` section. - -/label ~"group::" ~"section::" ~"Category::" ~"GitLab Core"/~"GitLab Premium"/~"GitLab Ultimate" - --> + +/label ~feature::addition ~"group::" ~"section::" ~"Category:" ~"GitLab Core"/~"GitLab Premium"/~"GitLab Ultimate" diff --git a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md index be6fef40f3a..560b1722348 100644 --- a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md +++ b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md @@ -1,6 +1,6 @@ <!-- -This template is based on a model named `CoolWidget`. +This template is based on a model named `CoolWidget`. To adapt this template, find and replace the following tokens: @@ -34,6 +34,9 @@ There are three main sections below. It is a good idea to structure your merge r It is also a good idea to first open a proof-of-concept merge request. It can be helpful for working out kinks and getting initial support and feedback from the Geo team. As an example, see the [Proof of Concept to replicate Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56423). +You can look into the following example for implementing replication/verification for a new Git repository type: +- [Add snippet repository verification](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56596) + ### Modify database schemas to prepare to add Geo support for Cool Widgets You might do this section in its own merge request, but it is not required. @@ -342,39 +345,6 @@ That's all of the required database changes. - [ ] Implement `CoolWidget.replicables_for_current_secondary` above. - [ ] Ensure `CoolWidget.replicables_for_current_secondary` is well-tested. Search the codebase for `replicables_for_current_secondary` to find examples of parameterized table specs. You may need to add more `FactoryBot` traits. -- [ ] If you are using a separate table `cool_widget_states` to track verification state on the Geo primary site, then: - - [ ] Do not include `::Gitlab::Geo::VerificationState` on the `CoolWidget` class. - - [ ] Add the following lines to the `cool_widget_state.rb` model: - - ```ruby - class CoolWidgetState < ApplicationRecord - ... - self.primary_key = :cool_widget_id - - include ::Gitlab::Geo::VerificationState - - belongs_to :cool_widget, inverse_of: :cool_widget_state - ... - end - ``` - - - [ ] Add the following lines to the `cool_widget` model: - - ```ruby - class CoolWidget < ApplicationRecord - ... - has_one :cool_widget_state, inverse_of: :cool_widget - - delegate :verification_retry_at, :verification_retry_at=, - :verified_at, :verified_at=, - :verification_checksum, :verification_checksum=, - :verification_failure, :verification_failure=, - :verification_retry_count, :verification_retry_count=, - to: :cool_widget_state - ... - end - ``` - - [ ] Create `ee/app/replicators/geo/cool_widget_replicator.rb`. Implement the `#repository` method which should return a `<Repository>` instance, and implement the class method `.model` to return the `CoolWidget` class: ```ruby @@ -383,6 +353,7 @@ That's all of the required database changes. module Geo class CoolWidgetReplicator < Gitlab::Geo::Replicator include ::Geo::RepositoryReplicatorStrategy + extend ::Gitlab::Utils::Override def self.model ::CoolWidget @@ -493,7 +464,7 @@ That's all of the required database changes. FactoryBot.define do factory :geo_cool_widget_registry, class: 'Geo::CoolWidgetRegistry' do - cool_widget + cool_widget # This association should have data, like a file or repository state { Geo::CoolWidgetRegistry.state_value(:pending) } trait :synced do @@ -542,6 +513,119 @@ That's all of the required database changes. end ``` +- [ ] Add the following to `spec/factories/cool_widgets.rb`: + + ```ruby + trait(:verification_succeeded) do + with_file + verification_checksum { 'abc' } + verification_state { CoolWidget.verification_state_value(:verification_succeeded) } + end + + trait(:verification_failed) do + with_file + verification_failure { 'Could not calculate the checksum' } + verification_state { CoolWidget.verification_state_value(:verification_failed) } + end + ``` + +- [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. + +##### If you added verification state fields to a separate table (option 2 above), then you need to make additional model and factory changes + +If you did not add verification state fields to a separate table, `cool_widget_states`, then skip to [Step 2. Implement metrics gathering](#step-2-implement-metrics-gathering). + +Otherwise, you can follow [the example of Merge Request Diffs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309). + +- [ ] Add a `Geo::CoolWidgetState` model in `ee/app/models/ee/geo/cool_widget_state.rb`: + + ``` ruby + module Geo + class CoolWidgetState < ApplicationRecord + self.primary_key = :cool_widget_id + + belongs_to :cool_widget, inverse_of: :cool_widget_state + end + end + ``` + +- [ ] Add a `factory` for `cool_widget_state`, in `ee/spec/factories/geo/cool_widget_states.rb`: + + ``` ruby + # frozen_string_literal: true + + FactoryBot.define do + factory :geo_cool_widget_state, class: 'Geo::CoolWidgetState' do + cool_widget + + trait(:checksummed) do + verification_checksum { 'abc' } + end + + trait(:checksum_failure) do + verification_failure { 'Could not calculate the checksum' } + end + end + end + ``` + +- [ ] Add the following lines to the `cool_widget` model to accomplish some important tasks: + - Include the `::Gitlab::Geo::VerificationState` concern. + - Delegate verification related methods to the `cool_widget_state` model. + - Override some scopes to use the `cool_widget_states` table instead of the model table, for verification. + - Override some methods to use the `cool_widget_states` table in verification related queries. + + ```ruby + class CoolWidget < ApplicationRecord + ... + include ::Gitlab::Geo::VerificationState + + has_one :cool_widget_state, autosave: true, inverse_of: :cool_widget, class_name: 'Geo::CoolWidgetState' + + delegate :verification_retry_at, :verification_retry_at=, + :verified_at, :verified_at=, + :verification_checksum, :verification_checksum=, + :verification_failure, :verification_failure=, + :verification_retry_count, :verification_retry_count=, + :verification_state=, :verification_state, + :verification_started_at=, :verification_started_at, + to: :cool_widget_state + ... + + scope :with_verification_state, ->(state) { joins(:cool_widget_state).where(cool_widget_states: { verification_state: verification_state_value(state) }) } + scope :checksummed, -> { joins(:cool_widget_state).where.not(cool_widget_states: { verification_checksum: nil } ) } + scope :not_checksummed, -> { joins(:cool_widget_state).where(cool_widget_states: { verification_checksum: nil } ) } + + ... + + class_methods do + extend ::Gitlab::Utils::Override + ... + override :verification_state_table_name + def verification_state_table_name + 'cool_widget_states' + end + + override :verification_state_model_key + def verification_state_model_key + 'cool_widget_id' + end + + override :verification_arel_table + def verification_arel_table + CoolWidgetState.arel_table + end + end + ... + + def cool_widget_state + super || build_cool_widget_state + end + + ... + end + ``` + #### Step 2. Implement metrics gathering Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` for display in the UI, and sent to Prometheus: @@ -578,24 +662,6 @@ Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` Geo::CoolWidgetReplicator | :cool_widget | :geo_cool_widget_registry ``` -- [ ] Add the following to `spec/factories/cool_widgets.rb`: - - ```ruby - trait(:verification_succeeded) do - with_file - verification_checksum { 'abc' } - verification_state { CoolWidget.verification_state_value(:verification_succeeded) } - end - - trait(:verification_failed) do - with_file - verification_failure { 'Could not calculate the checksum' } - verification_state { CoolWidget.verification_state_value(:verification_failed) } - end - ``` - -- [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. - Cool Widget replication and verification metrics should now be available in the API, the `Admin > Geo > Nodes` view, and Prometheus. #### Step 3. Implement the GraphQL API @@ -736,6 +802,8 @@ Individual Cool Widget replication and verification data should now be available module Geo class CoolWidgetReplicator < Gitlab::Geo::Replicator ... + # REMOVE THIS LINE IF IT IS NO LONGER NEEDED + extend ::Gitlab::Utils::Override # REMOVE THIS METHOD def self.replication_enabled_by_default? diff --git a/.gitlab/issue_templates/Geo Replicate a new blob type.md b/.gitlab/issue_templates/Geo Replicate a new blob type.md index c7bbb6b937c..58b08f490f7 100644 --- a/.gitlab/issue_templates/Geo Replicate a new blob type.md +++ b/.gitlab/issue_templates/Geo Replicate a new blob type.md @@ -1,6 +1,6 @@ <!-- -This template is based on a model named `CoolWidget`. +This template is based on a model named `CoolWidget`. To adapt this template, find and replace the following tokens: @@ -34,6 +34,10 @@ There are three main sections below. It is a good idea to structure your merge r It is also a good idea to first open a proof-of-concept merge request. It can be helpful for working out kinks and getting initial support and feedback from the Geo team. As an example, see the [Proof of Concept to replicate Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56423). +You can look into the following examples of MRs for implementing replication/verification for a new blob type: +- [Add db changes](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60935) and [add verification for MR diffs using SSF](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309) +- [Verify Terraform state versions](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58800) + ### Modify database schemas to prepare to add Geo support for Cool Widgets You might do this section in its own merge request, but it is not required. @@ -331,39 +335,6 @@ That's all of the required database changes. - [ ] Implement `CoolWidget.replicables_for_current_secondary` above. - [ ] Ensure `CoolWidget.replicables_for_current_secondary` is well-tested. Search the codebase for `replicables_for_current_secondary` to find examples of parameterized table specs. You may need to add more `FactoryBot` traits. -- [ ] If you are using a separate table `cool_widget_states` to track verification state on the Geo primary site, then: - - [ ] Do not include `::Gitlab::Geo::VerificationState` on the `CoolWidget` class. - - [ ] Add the following lines to the `cool_widget_state.rb` model: - - ```ruby - class CoolWidgetState < ApplicationRecord - ... - self.primary_key = :cool_widget_id - - include ::Gitlab::Geo::VerificationState - - belongs_to :cool_widget, inverse_of: :cool_widget_state - ... - end - ``` - - - [ ] Add the following lines to the `cool_widget` model: - - ```ruby - class CoolWidget < ApplicationRecord - ... - has_one :cool_widget_state, inverse_of: :cool_widget - - delegate :verification_retry_at, :verification_retry_at=, - :verified_at, :verified_at=, - :verification_checksum, :verification_checksum=, - :verification_failure, :verification_failure=, - :verification_retry_count, :verification_retry_count=, - to: :cool_widget_state - ... - end - ``` - - [ ] Create `ee/app/replicators/geo/cool_widget_replicator.rb`. Implement the `#carrierwave_uploader` method which should return a `CarrierWave::Uploader`, and implement the class method `.model` to return the `CoolWidget` class: ```ruby @@ -372,6 +343,7 @@ That's all of the required database changes. module Geo class CoolWidgetReplicator < Gitlab::Geo::Replicator include ::Geo::BlobReplicatorStrategy + extend ::Gitlab::Utils::Override def self.model ::CoolWidget @@ -459,7 +431,7 @@ That's all of the required database changes. FactoryBot.define do factory :geo_cool_widget_registry, class: 'Geo::CoolWidgetRegistry' do - cool_widget + cool_widget # This association should have data, like a file or repository state { Geo::CoolWidgetRegistry.state_value(:pending) } trait :synced do @@ -508,6 +480,119 @@ That's all of the required database changes. end ``` +- [ ] Add the following to `spec/factories/cool_widgets.rb`: + + ```ruby + trait(:verification_succeeded) do + with_file + verification_checksum { 'abc' } + verification_state { CoolWidget.verification_state_value(:verification_succeeded) } + end + + trait(:verification_failed) do + with_file + verification_failure { 'Could not calculate the checksum' } + verification_state { CoolWidget.verification_state_value(:verification_failed) } + end + ``` + +- [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. + +##### If you added verification state fields to a separate table (option 2 above), then you need to make additional model and factory changes + +If you did not add verification state fields to a separate table, `cool_widget_states`, then skip to [Step 2. Implement metrics gathering](#step-2-implement-metrics-gathering). + +Otherwise, you can follow [the example of Merge Request Diffs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309). + +- [ ] Add a `Geo::CoolWidgetState` model in `ee/app/models/ee/geo/cool_widget_state.rb`: + + ``` ruby + module Geo + class CoolWidgetState < ApplicationRecord + self.primary_key = :cool_widget_id + + belongs_to :cool_widget, inverse_of: :cool_widget_state + end + end + ``` + +- [ ] Add a `factory` for `cool_widget_state`, in `ee/spec/factories/geo/cool_widget_states.rb`: + + ``` ruby + # frozen_string_literal: true + + FactoryBot.define do + factory :geo_cool_widget_state, class: 'Geo::CoolWidgetState' do + cool_widget + + trait(:checksummed) do + verification_checksum { 'abc' } + end + + trait(:checksum_failure) do + verification_failure { 'Could not calculate the checksum' } + end + end + end + ``` + +- [ ] Add the following lines to the `cool_widget` model to accomplish some important tasks: + - Include the `::Gitlab::Geo::VerificationState` concern. + - Delegate verification related methods to the `cool_widget_state` model. + - Override some scopes to use the `cool_widget_states` table instead of the model table, for verification. + - Override some methods to use the `cool_widget_states` table in verification related queries. + + ```ruby + class CoolWidget < ApplicationRecord + ... + include ::Gitlab::Geo::VerificationState + + has_one :cool_widget_state, autosave: true, inverse_of: :cool_widget, class_name: 'Geo::CoolWidgetState' + + delegate :verification_retry_at, :verification_retry_at=, + :verified_at, :verified_at=, + :verification_checksum, :verification_checksum=, + :verification_failure, :verification_failure=, + :verification_retry_count, :verification_retry_count=, + :verification_state=, :verification_state, + :verification_started_at=, :verification_started_at, + to: :cool_widget_state + ... + + scope :with_verification_state, ->(state) { joins(:cool_widget_state).where(cool_widget_states: { verification_state: verification_state_value(state) }) } + scope :checksummed, -> { joins(:cool_widget_state).where.not(cool_widget_states: { verification_checksum: nil } ) } + scope :not_checksummed, -> { joins(:cool_widget_state).where(cool_widget_states: { verification_checksum: nil } ) } + + ... + + class_methods do + extend ::Gitlab::Utils::Override + ... + override :verification_state_table_name + def verification_state_table_name + 'cool_widget_states' + end + + override :verification_state_model_key + def verification_state_model_key + 'cool_widget_id' + end + + override :verification_arel_table + def verification_arel_table + CoolWidgetState.arel_table + end + end + ... + + def cool_widget_state + super || build_cool_widget_state + end + + ... + end + ``` + #### Step 2. Implement metrics gathering Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` for display in the UI, and sent to Prometheus: @@ -544,24 +629,6 @@ Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` Geo::CoolWidgetReplicator | :cool_widget | :geo_cool_widget_registry ``` -- [ ] Add the following to `spec/factories/cool_widgets.rb`: - - ```ruby - trait(:verification_succeeded) do - with_file - verification_checksum { 'abc' } - verification_state { CoolWidget.verification_state_value(:verification_succeeded) } - end - - trait(:verification_failed) do - with_file - verification_failure { 'Could not calculate the checksum' } - verification_state { CoolWidget.verification_state_value(:verification_failed) } - end - ``` - -- [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. - Cool Widget replication and verification metrics should now be available in the API, the `Admin > Geo > Nodes` view, and Prometheus. #### Step 3. Implement the GraphQL API @@ -702,7 +769,10 @@ Individual Cool Widget replication and verification data should now be available module Geo class CoolWidgetReplicator < Gitlab::Geo::Replicator ... + # REMOVE THIS LINE IF IT IS NO LONGER NEEDED + extend ::Gitlab::Utils::Override + ... # REMOVE THIS METHOD def self.replication_enabled_by_default? false diff --git a/.gitlab/issue_templates/Security developer workflow.md b/.gitlab/issue_templates/Security developer workflow.md index 25825fc8888..99149442509 100644 --- a/.gitlab/issue_templates/Security developer workflow.md +++ b/.gitlab/issue_templates/Security developer workflow.md @@ -10,10 +10,11 @@ Set the title to: `Description of the original issue` - [ ] Read the [security process for developers] if you are not familiar with it. - Verify if the issue you're working on `gitlab-org/gitlab` is confidential, if it's public fix should be placed on GitLab canonical and no backports are required. -- [ ] Mark this [issue as linked] to the Security Release Tracking Issue. You can find it on the topic of the `#releases` Slack channel. +- [ ] **IMPORTANT**: Mark this [issue as linked] to the Security Release Tracking Issue. You can find it on the topic of the `#releases` Slack channel. This issue +MUST be linked for the release bot to know that the associated merge requests should be merged for this security release. - Fill out the [Links section](#links): - [ ] Next to **Issue on GitLab**, add a link to the `gitlab-org/gitlab` issue that describes the security vulnerability. -- [ ] Add one of the `~severity::x` labels to the issue and all associated merge requests. +- [ ] Add one of the `~severity::x` labels to the issue and all associated merge requests. ## Development diff --git a/.gitlab/issue_templates/Snowplow event tracking.md b/.gitlab/issue_templates/Snowplow event tracking.md index 4a906b61378..ee8755d3292 100644 --- a/.gitlab/issue_templates/Snowplow event tracking.md +++ b/.gitlab/issue_templates/Snowplow event tracking.md @@ -4,7 +4,7 @@ * Snowplow is currently not used for self-hosted instances of GitLab - Self-hosted still rely on usage ping for product analytics - Snowplow is used for GitLab SaaS * You do not need to create an issue to track generic front-end events, such as All page views, sessions, link clicks, some button clicks, etc. * What you should capture are specific events with defined business logic. For example, when a user creates an incident by escalating an existing alert, or when a user creates and pushes up a new Node package to the NPM registry. -* For more details read https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/ +* For more details read https://about.gitlab.com/handbook/business-technology/data-team/programs/data-for-product-managers/ --> <!-- diff --git a/.gitlab/merge_request_templates/Documentation.md b/.gitlab/merge_request_templates/Documentation.md index 9c6b5a73b4f..99ad233c7e0 100644 --- a/.gitlab/merge_request_templates/Documentation.md +++ b/.gitlab/merge_request_templates/Documentation.md @@ -13,62 +13,29 @@ <!-- Link related issues below. --> -## Author's checklist (required) +## Author's checklist - [ ] Follow the [Documentation Guidelines](https://docs.gitlab.com/ee/development/documentation/) and [Style Guide](https://docs.gitlab.com/ee/development/documentation/styleguide/). -- If you have **Developer** permissions or higher: - - [ ] Ensure that the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#product-tier-badges) is added to doc's `h1`. - - [ ] Apply the ~documentation label, plus: - - The corresponding DevOps stage and group labels, if applicable. - - ~"development guidelines" when changing docs under `doc/development/*`, `CONTRIBUTING.md`, or `README.md`. - - ~"development guidelines" and ~"Documentation guidelines" when changing docs under `development/documentation/*`. - - ~"development guidelines" and ~"Description templates (.gitlab/\*)" when creating/updating issue and MR description templates. - - [ ] [Request a review](https://docs.gitlab.com/ee/development/code_review.html#dogfooding-the-reviewers-feature) - from the [designated Technical Writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments). +- [ ] Ensure that the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#product-tier-badges) is added to doc's `h1`. +- [ ] [Request a review](https://docs.gitlab.com/ee/development/code_review.html#dogfooding-the-reviewers-feature) based on the documentation page's metadata and [associated Technical Writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments). -/label ~documentation -/assign me - -Do not add the ~"feature", ~"frontend", ~"backend", ~"bug", or ~"database" labels if you are only updating documentation. These labels will cause the MR to be added to code verification QA issues. - -When applicable: - -- [ ] Update the [permissions table](https://docs.gitlab.com/ee/user/permissions.html). -- [ ] Link docs to and from the higher-level index page, plus other related docs where helpful. -- [ ] Add the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#product-tier-badges) accordingly. -- [ ] Add [GitLab's version history note(s)](https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#gitlab-versions). -- [ ] Add/update the [feature flag section](https://docs.gitlab.com/ee/development/documentation/feature_flags.html). +To avoid having this MR be added to code verification QA issues, don't add these labels: ~"feature", ~"frontend", ~"backend", ~"bug", or ~"database" ## Review checklist -All reviewers can help ensure accuracy, clarity, completeness, and adherence to the [Documentation Guidelines](https://docs.gitlab.com/ee/development/documentation/) and [Style Guide](https://docs.gitlab.com/ee/development/documentation/styleguide/). - -**1. Primary Reviewer** - -* [ ] Review by a code reviewer or other selected colleague to confirm accuracy, clarity, and completeness. This can be skipped for minor fixes without substantive content changes. - -**2. Technical Writer** +Documentation-related MRs should be reviewed by a Technical Writer for a non-blocking review, based on [Documentation Guidelines](https://docs.gitlab.com/ee/development/documentation/) and the [Style Guide](https://docs.gitlab.com/ee/development/documentation/styleguide/). -- [ ] Technical writer review. If not requested for this MR, must be scheduled post-merge. To request for this MR, assign the writer listed for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). - - [ ] Ensure docs metadata are present and up-to-date. - - [ ] Ensure ~"Technical Writing" and ~"documentation" are added. - - [ ] Add the corresponding `docs::` [scoped label](https://gitlab.com/groups/gitlab-org/-/labels?subscribed=&search=docs%3A%3A). - - [ ] If working on UI text, add the corresponding `UI Text` [scoped label](https://gitlab.com/groups/gitlab-org/-/labels?subscribed=&search=ui+text). - - [ ] Add ~"tw::doing" when starting work on the MR. - - [ ] Add ~"tw::finished" if Technical Writing team work on the MR is complete but it remains open. +- [ ] If the content requires it, ensure the information is reviewed by a subject matter expert. +- Technical writer review items: + - [ ] Ensure docs metadata is present and up-to-date. + - [ ] Ensure the appropriate [labels](https://about.gitlab.com/handbook/engineering/ux/technical-writing/workflow/#labels) are added to this MR. + - If relevant to this MR, ensure [content topic type](https://docs.gitlab.com/ee/development/documentation/structure.html) principles are in use, including: + - [ ] The headings should be something you'd do a Google search for. Instead of `Default behavior`, say something like `Default behavior when you close an issue`. + - [ ] The headings (other than the page title) should be active. Instead of `Configuring GDK`, say something like `Configure GDK`. + - [ ] Any task steps should be written as a numbered list. + - If the content still needs to be edited for topic types, you can create a follow-up issue with the ~"docs-technical-debt" label. +- [ ] Review by assigned maintainer, who can always request/require the above reviews. Maintainer's review can occur before or after a technical writer review. +- [ ] Ensure a release milestone is set. -For more information about labels, see [Technical Writing workflows - Labels](https://about.gitlab.com/handbook/engineering/ux/technical-writing/workflow/#labels). - -For suggestions that you are confident don't need to be reviewed, change them locally -and push a commit directly to save others from unneeded reviews. For example: - -- Clear typos, like `this is a typpo`. -- Minor issues, like single quotes instead of double quotes, Oxford commas, and periods. - -For more information, see our documentation on [Merging a merge request](https://docs.gitlab.com/ee/development/code_review.html#merging-a-merge-request). - -**3. Maintainer** - -1. [ ] Review by assigned maintainer, who can always request/require the above reviews. Maintainer's review can occur before or after a technical writer review. -1. [ ] Ensure a release milestone is set. -1. [ ] If there has not been a technical writer review, [create an issue for one using the Doc Review template](https://gitlab.com/gitlab-org/gitlab/issues/new?issuable_template=Doc%20Review). +/label ~documentation +/assign me diff --git a/.gitlab/merge_request_templates/Quarantine End to End Test.md b/.gitlab/merge_request_templates/Quarantine End to End Test.md index 5794a62df96..772167af3e9 100644 --- a/.gitlab/merge_request_templates/Quarantine End to End Test.md +++ b/.gitlab/merge_request_templates/Quarantine End to End Test.md @@ -21,7 +21,7 @@ the noise (due to constantly failing tests, flaky tests, and so on) so that new - [ ] Quarantine test check-list - [ ] Follow the [Quarantining Tests guide](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests). - [ ] Confirm the test has a [`quarantine:` tag with the specified quarantine type](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantined-test-types). - - [ ] Note if the test should be [quarantined for a specific environment](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/environment_selection.html#quarantining-a-test-for-a-specific-environment). + - [ ] Note if the test should be [quarantined for a specific environment](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/execution_context_selection.html#quarantine-a-test-for-a-specific-environment). - [ ] Dequarantine test check-list - [ ] Follow the [Dequarantining Tests guide](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#dequarantining-tests). - [ ] Confirm the test consistently passes on the target GitLab environment(s). diff --git a/.gitlab/merge_request_templates/Security Release.md b/.gitlab/merge_request_templates/Security Release.md index 77e8718c34f..33c0a5b98a8 100644 --- a/.gitlab/merge_request_templates/Security Release.md +++ b/.gitlab/merge_request_templates/Security Release.md @@ -30,8 +30,8 @@ See [the general developer security release guidelines](https://gitlab.com/gitla ## Maintainer checklist -- [ ] Correct milestone is applied and the title is matching across all backports -- [ ] Assigned to `@gitlab-release-tools-bot` with passing CI pipelines and **when all backports including the MR targeting master are ready.** +- [ ] Correct milestone is applied and the title is matching across all backports. +- [ ] Assigned to `@gitlab-release-tools-bot` with passing CI pipelines. /label ~security |