diff options
Diffstat (limited to '.gitlab')
30 files changed, 1875 insertions, 192 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 2eda1a890d9..6432640570f 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -135,7 +135,7 @@ /doc/api/invitations.md @aqualls /doc/api/experiments.md @aqualls /doc/development/experiment_guide/ @aqualls -/doc/development/snowplow.md @aqualls +/doc/development/snowplow/ @aqualls /doc/development/usage_ping/ @aqualls /doc/user/admin_area/license.md @aqualls @@ -193,7 +193,7 @@ Dangerfile @gl-quality/eng-prod /lib/gitlab/auth/ldap/ @dblessing @mkozono [Templates] -/lib/gitlab/ci/templates/ @nolith @shinya.maeda +/lib/gitlab/ci/templates/ @nolith @shinya.maeda @matteeyah /lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @DylanGriffith @mayra-cabrera @tkuah /lib/gitlab/ci/templates/Security/ @plafoucriere @gonzoyumo @twoodham @sethgitlab @@ -277,6 +277,7 @@ Dangerfile @gl-quality/eng-prod /lib/gitlab/experimentation/ @gitlab-org/growth/experiment-devs /lib/gitlab/experimentation.rb @gitlab-org/growth/experiment-devs /lib/gitlab/experimentation_logger.rb @gitlab-org/growth/experiment-devs +/ee/spec/requests/api/experiments_spec.rb @gitlab-org/growth/experiment-devs [Legal] /config/dependency_decisions.yml @gitlab-org/legal-reviewers diff --git a/.gitlab/changelog_config.yml b/.gitlab/changelog_config.yml new file mode 100644 index 00000000000..7aa18cc8f36 --- /dev/null +++ b/.gitlab/changelog_config.yml @@ -0,0 +1,38 @@ +--- +# Settings for generating changelogs using the GitLab API. See +# https://docs.gitlab.com/ee/api/repositories.html#generate-changelog-data for +# more information. +categories: + added: Added + fixed: Fixed + changed: Changed + deprecated: Deprecated + removed: Removed + security: Security + performance: Performance + other: Other +template: | + {% if categories %} + {% each categories %} + ### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %}) + + {% each entries %} + - [{{ title }}]({{ commit.reference }})\ + {% if author.contributor %} by {{ author.reference }}{% end %}\ + {% if commit.trailers.MR %}\ + ([merge request]({{ commit.trailers.MR }}))\ + {% else %}\ + {% if merge_request %}\ + ([merge request]({{ merge_request.reference }}))\ + {% end %}\ + {% end %}\ + {% if commit.trailers.EE %}\ + **GitLab Enterprise Edition**\ + {% end %} + + {% end %} + + {% end %} + {% else %} + No changes. + {% end %} diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index e6c3e7598d3..4e352472047 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -1,6 +1,8 @@ -# This image is used by the `review-qa-*` jobs. Not currently used by the `omnibus-gitlab` pipelines which rebuild this -# image, e.g. https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/jobs/587107399, which we could probably avoid. -# See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5429. +# This image is used by the `review-qa-*` jobs. The image name is also passed to the downstream `omnibus-gitlab-mirror` pipeline +# triggered by `package-and-qa` so that it doesn't have to rebuild it a second time. The downstream `omnibus-gitlab-mirror` pipeline +# itself passes the image name to the `gitlab-qa-mirror` pipeline so that it can use it instead of inferring an end-to-end image +# from the GitLab image built by the downstream `omnibus-gitlab-mirror` pipeline. +# See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#testing-code-in-merge-requests for more details. build-qa-image: extends: - .use-kaniko diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml index 324c8615083..475cbca3156 100644 --- a/.gitlab/ci/cache-repo.gitlab-ci.yml +++ b/.gitlab/ci/cache-repo.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Builds a cached .tar.gz of the master branch with full history and +# Builds a cached .tar.gz of the $CI_DEFAULT_BRANCH branch with full history and # uploads it to Google Cloud Storage. This archive is downloaded by a # script defined by a CI/CD variable named CI_PRE_CLONE_SCRIPT. This has # two benefits: @@ -41,6 +41,7 @@ cache-repo: cd $CI_PROJECT_NAME; time git repack -d; echo "Archiving $CI_PROJECT_NAME into /tmp/$SHALLOW_CLONE_TAR_FILENAME."; + time git remote rm origin; time tar cf /tmp/$SHALLOW_CLONE_TAR_FILENAME .; echo "GZipping /tmp/$SHALLOW_CLONE_TAR_FILENAME."; time gzip /tmp/$SHALLOW_CLONE_TAR_FILENAME; @@ -52,7 +53,9 @@ cache-repo: echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME."; time git clone --progress $CI_REPOSITORY_URL $CI_PROJECT_NAME; cd $CI_PROJECT_NAME; + time git repack -d; echo "Archiving $CI_PROJECT_NAME into /tmp/$FULL_CLONE_TAR_FILENAME."; + time git remote rm origin; time tar cf /tmp/$FULL_CLONE_TAR_FILENAME .; echo "GZipping /tmp/$FULL_CLONE_TAR_FILENAME."; time gzip /tmp/$FULL_CLONE_TAR_FILENAME; diff --git a/.gitlab/ci/dast.gitlab-ci.yml b/.gitlab/ci/dast.gitlab-ci.yml index 33778b9cbd0..a8a201bd1fa 100644 --- a/.gitlab/ci/dast.gitlab-ci.yml +++ b/.gitlab/ci/dast.gitlab-ci.yml @@ -97,7 +97,7 @@ DAST-fullscan-ruleset5: variables: DAST_USERNAME: "user5" script: - - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10010 | enable_rule 10011 | enable_rule 10015 | enable_rule 10017 | enable_rule 10019) + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10010 | enable_rule 10011 | enable_rule 10017 | enable_rule 10019) - echo $DAST_EXCLUDE_RULES - /analyze -t $DAST_WEBSITE -d diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index b42b32ea44d..8f70127be65 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -10,17 +10,18 @@ # because some repos are private and CI_JOB_TOKEN cannot access files. # See https://gitlab.com/gitlab-org/gitlab/issues/191273 GIT_DEPTH: 1 + # By default, deploy the Review App using the `master` branch of the `gitlab-org/gitlab-docs` project + DOCS_BRANCH: master environment: - name: review-docs/$DOCS_GITLAB_REPO_SUFFIX-$CI_MERGE_REQUEST_IID + name: review-docs/mr-${CI_MERGE_REQUEST_IID} # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables # Discussion: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/14236/diffs#note_40140693 auto_stop_in: 2 weeks - url: http://docs-preview-$DOCS_GITLAB_REPO_SUFFIX-$CI_MERGE_REQUEST_IID.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX + url: http://${DOCS_BRANCH}-${DOCS_GITLAB_REPO_SUFFIX}-${CI_MERGE_REQUEST_IID}.${DOCS_REVIEW_APPS_DOMAIN}/${DOCS_GITLAB_REPO_SUFFIX} on_stop: review-docs-cleanup before_script: - - apk add --update openssl - - gem install httparty --no-document --version 0.17.3 - - gem install gitlab --no-document --version 4.13.0 + - source ./scripts/utils.sh + - install_gitlab_gem # Always trigger a docs build in gitlab-docs only on docs-only branches. # Useful to preview the docs changes live. @@ -33,7 +34,7 @@ review-docs-deploy: review-docs-cleanup: extends: .review-docs environment: - name: review-docs/$DOCS_GITLAB_REPO_SUFFIX-$CI_MERGE_REQUEST_IID + name: review-docs/mr-${CI_MERGE_REQUEST_IID} action: stop script: - ./scripts/trigger-build docs cleanup @@ -64,10 +65,8 @@ docs-lint links: - cd /tmp/gitlab-docs # Build HTML from Markdown - bundle exec nanoc - # Check the internal links - - bundle exec nanoc check internal_links - # Check the internal anchor links - - bundle exec nanoc check internal_anchors + # Check the internal links and anchors (in parallel) + - "parallel time bundle exec nanoc check ::: internal_links internal_anchors" ui-docs-links lint: extends: diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 910a58bcd0e..c39f5cdce9c 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -1,22 +1,15 @@ -.frontend-base: - extends: - - .default-retry - - .default-before_script - variables: - SETUP_DB: "false" - # we override the max_old_space_size to prevent OOM errors - NODE_OPTIONS: --max_old_space_size=3584 - .yarn-install: &yarn-install - source scripts/utils.sh - run_timed_command "retry yarn install --frozen-lockfile" .compile-assets-base: extends: - - .frontend-base + - .default-retry + - .default-before_script - .assets-compile-cache - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2-git-2.29-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.34 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 variables: + SETUP_DB: "false" WEBPACK_VENDOR_DLL: "true" stage: prepare script: @@ -93,13 +86,13 @@ update-yarn-cache: .frontend-fixtures-base: extends: - - .frontend-base + - .default-retry + - .default-before_script - .rails-cache - .use-pg11 stage: fixtures needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"] variables: - SETUP_DB: "true" WEBPACK_VENDOR_DLL: "true" script: - run_timed_command "gem install knapsack --no-document" @@ -151,10 +144,8 @@ graphql-schema-dump: .frontend-test-base: extends: - - .frontend-base + - .default-retry - .yarn-cache - variables: - USE_BUNDLE_INSTALL: "false" stage: test eslint-as-if-foss: @@ -246,7 +237,7 @@ coverage-frontend: extends: - .default-retry - .yarn-cache - - .frontend:rules:ee-mr-and-master-only + - .frontend:rules:ee-mr-and-default-branch-only needs: ["jest"] stage: post-test before_script: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index de4b609098d..891457afe6e 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -27,7 +27,7 @@ .rails-cache: cache: - key: "rails-v4" + key: "rails-v5" paths: - vendor/ruby/ - vendor/gitaly-ruby/ @@ -87,7 +87,7 @@ policy: pull .use-pg11: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.29-lfs-2.9-chrome-87-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -96,7 +96,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg12: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.29-lfs-2.9-chrome-87-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -105,7 +105,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg11-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.29-lfs-2.9-chrome-87-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -116,7 +116,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg12-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.29-lfs-2.9-chrome-87-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index ef6c9b9c8ff..3e5639e4d69 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -44,8 +44,6 @@ memory-on-boot: NODE_ENV: "production" RAILS_ENV: "production" SETUP_DB: "true" - # we override the max_old_space_size to prevent OOM errors - NODE_OPTIONS: --max_old_space_size=3584 script: - PATH_TO_HIT="/users/sign_in" CUT_OFF=0.3 bundle exec derailed exec perf:mem >> 'tmp/memory_on_boot.txt' - scripts/generate-memory-metrics-on-boot tmp/memory_on_boot.txt >> 'tmp/memory_on_boot_metrics.txt' diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index e1ddefca99e..064aa5a8351 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -142,7 +142,7 @@ ############################ ####################################################### -# EE/FOSS: default refs (MRs, master, schedules) jobs # +# EE/FOSS: default refs (MRs, default branch, schedules) jobs # setup-test-env: extends: - .rails-job-base @@ -183,6 +183,7 @@ setup-test-env: - tmp/tests/gitlab-workhorse/gitlab-workhorse - tmp/tests/gitlab-workhorse/gitlab-resize-image - tmp/tests/gitlab-workhorse/config.toml + - tmp/tests/gitlab-workhorse/WORKHORSE_TREE - tmp/tests/repositories/ - tmp/tests/second_storage/ when: always @@ -256,17 +257,6 @@ static-analysis: - run_timed_command "retry yarn install --frozen-lockfile" - scripts/static-analysis -downtime_check: - extends: - - .rails-job-base - - .rails:rules:downtime_check - needs: [] - stage: test - variables: - SETUP_DB: "false" - script: - - bundle exec rake downtime_check - rspec migration pg11: extends: - .rspec-base-pg11 @@ -346,7 +336,7 @@ db:migrate:reset: db:check-schema: extends: - .db-job-base - - .rails:rules:ee-mr-and-master-only + - .rails:rules:ee-mr-and-default-branch-only script: - source scripts/schema_changed.sh @@ -358,32 +348,31 @@ db:check-migrations: - scripts/validate_migration_schema allow_failure: true -db:migrate-from-v12.10.0: +db:migrate-from-previous-major-version: extends: .db-job-base variables: + USE_BUNDLE_INSTALL: "false" SETUP_DB: "false" + PROJECT_TO_CHECKOUT: "gitlab-foss" + TAG_TO_CHECKOUT: "v12.10.14" script: - - export PROJECT_TO_CHECKOUT="gitlab" - - export TAG_TO_CHECKOUT="v12.10.0-ee" - - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab-foss"' - - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="v12.10.0"' + - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"' + - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="${TAG_TO_CHECKOUT}-ee"' - retry 'git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT' - git checkout -f FETCH_HEAD + # Patch Gemfile of the previous major version for compatibility. - sed -i -e "s/gem 'grpc', '~> 1.24.0'/gem 'grpc', '~> 1.30.2'/" Gemfile # Update gRPC for Ruby 2.7 - - sed -i -e "s/gem 'google-protobuf', '~> 3.8.0'/gem 'google-protobuf', '~> 3.12.0'/" Gemfile - - gem install bundler:1.17.3 - - bundle update google-protobuf grpc bootsnap - - bundle install $BUNDLE_INSTALL_FLAGS - - date + - sed -i -e "s/gem 'google-protobuf', '~> 3.8.0'/gem 'google-protobuf', '~> 3.12'/" Gemfile + - sed -i -e "s/gem 'nokogiri', '~> 1.10.5'/gem 'nokogiri', '~> 1.11.0'/" Gemfile + - sed -i -e "s/gem 'mimemagic', '~> 0.3.2'/gem 'ruby-magic', '~> 0.3.2'/" Gemfile + - run_timed_command "gem install bundler:1.17.3" + - run_timed_command "bundle update google-protobuf nokogiri grpc mimemagic bootsnap" + - run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}" - cp config/gitlab.yml.example config/gitlab.yml - - bundle exec rake db:drop db:create db:structure:load db:seed_fu - - date + - run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu" - git checkout -f $CI_COMMIT_SHA - - bundle install $BUNDLE_INSTALL_FLAGS - - date - - . scripts/prepare_build.sh - - date - - bundle exec rake db:migrate + - run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}" + - run_timed_command "bundle exec rake db:migrate" db:rollback: extends: .db-job-base @@ -535,11 +524,11 @@ rspec:feature-flags: run_timed_command "bundle exec scripts/used-feature-flags"; fi -# EE/FOSS: default refs (MRs, master, schedules) jobs # +# EE/FOSS: default refs (MRs, default branch, schedules) jobs # ####################################################### ################################################## -# EE: default refs (MRs, master, schedules) jobs # +# EE: default refs (MRs, default branch, schedules) jobs # rspec migration pg11-as-if-foss: extends: - .rspec-base-pg11-as-if-foss @@ -682,81 +671,81 @@ db:rollback geo: script: - bundle exec rake geo:db:migrate VERSION=20170627195211 - bundle exec rake geo:db:migrate -# EE: default refs (MRs, master, schedules) jobs # +# EE: default refs (MRs, default branch, schedules) jobs # ################################################## ########################################## -# EE/FOSS: master nightly scheduled jobs # +# EE/FOSS: default branch nightly scheduled jobs # rspec migration pg12: extends: - .rspec-base-pg12 - .rspec-base-migration - - .rails:rules:master-schedule-nightly--code-backstage + - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-migration-parallel rspec unit pg12: extends: - .rspec-base-pg12 - - .rails:rules:master-schedule-nightly--code-backstage + - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-unit-parallel rspec integration pg12: extends: - .rspec-base-pg12 - - .rails:rules:master-schedule-nightly--code-backstage + - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-integration-parallel rspec system pg12: extends: - .rspec-base-pg12 - - .rails:rules:master-schedule-nightly--code-backstage + - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-system-parallel -# EE/FOSS: master nightly scheduled jobs # +# EE/FOSS: default branch nightly scheduled jobs # ########################################## ##################################### -# EE: master nightly scheduled jobs # +# EE: default branch nightly scheduled jobs # rspec-ee migration pg12: extends: - .rspec-ee-base-pg12 - .rspec-base-migration - - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-migration-parallel rspec-ee unit pg12: extends: - .rspec-ee-base-pg12 - - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-parallel rspec-ee integration pg12: extends: - .rspec-ee-base-pg12 - - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-integration-parallel rspec-ee system pg12: extends: - .rspec-ee-base-pg12 - - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-system-parallel rspec-ee unit pg12 geo: extends: - .rspec-ee-base-geo-pg12 - - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-geo-parallel rspec-ee integration pg12 geo: extends: - .rspec-ee-base-geo-pg12 - - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only rspec-ee system pg12 geo: extends: - .rspec-ee-base-geo-pg12 - - .rails:rules:master-schedule-nightly--code-backstage-ee-only -# EE: master nightly scheduled jobs # + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only +# EE: default branch nightly scheduled jobs # ##################################### ################################################## @@ -799,7 +788,7 @@ fail-pipeline-early: GIT_DEPTH: 1 before_script: - source scripts/utils.sh - - install_api_client_dependencies_with_apt + - install_gitlab_gem script: - fail_pipeline_early # EE: Canonical MR pipelines diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 76191a923bf..e1037edc325 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -29,7 +29,6 @@ review-build-cng: stage: review-prepare before_script: - source ./scripts/utils.sh - - install_api_client_dependencies_with_apk - install_gitlab_gem needs: - job: compile-production-assets @@ -161,7 +160,7 @@ review-qa-smoke: review-qa-all: extends: - .review-qa-base - - .review:rules:mr-only-manual + - .review:rules:review-qa-all parallel: 5 script: - export KNAPSACK_REPORT_PATH=knapsack/master_report.json @@ -198,7 +197,7 @@ review-performance: parallel-spec-reports: extends: - - .review:rules:mr-only-manual + - .review:rules:review-qa-all image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine stage: post-qa dependencies: ["review-qa-all"] @@ -234,7 +233,15 @@ danger-review: - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --with danger" - run_timed_command "retry yarn install --frozen-lockfile" script: - - run_timed_command "bundle exec danger --fail-on-errors=true --verbose" + - > + if [ -z "$DANGER_GITLAB_API_TOKEN" ]; then + # Force danger to skip CI source GitLab and fallback to "local only git repo". + unset GITLAB_CI + # We need to base SHA to help danger determine the base commit for this shallow clone. + run_timed_command "bundle exec danger dry_run --fail-on-errors=true --verbose --base='$CI_MERGE_REQUEST_DIFF_BASE_SHA'" + else + run_timed_command "bundle exec danger --fail-on-errors=true --verbose" + fi update-danger-review-cache: extends: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index e76b0f2d07f..c2d16582a68 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -11,25 +11,25 @@ if: '$CI_PROJECT_NAME != "gitlab-foss" && $CI_PROJECT_NAME != "gitlab-ce" && $CI_PROJECT_NAME != "gitlabhq"' .if-default-refs: &if-default-refs - if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "main" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG || $FORCE_GITLAB_CI' + if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG || $FORCE_GITLAB_CI' -.if-master-refs: &if-master-refs - if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "main"' +.if-default-branch-refs: &if-default-branch-refs + if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' -.if-master-push: &if-master-push - if: '($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_REF_NAME == "main") && $CI_PIPELINE_SOURCE == "push"' +.if-default-branch-push: &if-default-branch-push + if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' -.if-master-schedule-2-hourly: &if-master-schedule-2-hourly - if: '($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_REF_NAME == "main") && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' +.if-default-branch-schedule-2-hourly: &if-default-branch-schedule-2-hourly + if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' -.if-master-schedule-nightly: &if-master-schedule-nightly - if: '($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_REF_NAME == "main") && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "nightly"' +.if-default-branch-schedule-nightly: &if-default-branch-schedule-nightly + if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "nightly"' .if-auto-deploy-branches: &if-auto-deploy-branches if: '$CI_COMMIT_BRANCH =~ /^\d+-\d+-auto-deploy-\d+$/' -.if-master-or-tag: &if-master-or-tag - if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "main" || $CI_COMMIT_TAG' +.if-default-branch-or-tag: &if-default-branch-or-tag + if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG' .if-merge-request: &if-merge-request if: '$CI_MERGE_REQUEST_IID' @@ -52,8 +52,8 @@ .if-dot-com-gitlab-org-schedule: &if-dot-com-gitlab-org-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"' -.if-dot-com-gitlab-org-master: &if-dot-com-gitlab-org-master - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && ($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "main")' +.if-dot-com-gitlab-org-default-branch: &if-dot-com-gitlab-org-default-branch + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' .if-dot-com-gitlab-org-merge-request: &if-dot-com-gitlab-org-merge-request if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID' @@ -101,6 +101,7 @@ - ".gitlab/ci/frontend.gitlab-ci.yml" - ".gitlab/ci/build-images.gitlab-ci.yml" - ".gitlab/ci/review.gitlab-ci.yml" + - "scripts/review_apps/base-config.yaml" - "scripts/trigger-build" .ci-qa-patterns: &ci-qa-patterns @@ -131,6 +132,14 @@ - "config/webpack.config.js" - "config/helpers/*.js" +.frontend-build-patterns: &frontend-build-patterns + - "{package.json,yarn.lock}" + - "babel.config.js" + - "config/webpack.config.js" + - "config/**/*.js" + - "vendor/assets/**/*" + - "{,ee/}app/assets/**/*" + .frontend-patterns: &frontend-patterns - "{package.json,yarn.lock}" - "babel.config.js" @@ -293,7 +302,7 @@ ################ .shared:rules:update-cache: rules: - - <<: *if-master-schedule-2-hourly + - <<: *if-default-branch-schedule-2-hourly - <<: *if-security-schedule - <<: *if-merge-request-title-update-caches @@ -314,6 +323,7 @@ rules: - <<: *if-not-canonical-namespace when: never + - <<: *if-auto-deploy-branches - changes: *ci-build-images-patterns - changes: *code-qa-patterns @@ -394,8 +404,8 @@ rules: - <<: *if-not-canonical-namespace when: never - - <<: *if-default-refs - changes: *code-qa-patterns + - <<: *if-auto-deploy-branches + - changes: *code-qa-patterns .frontend:rules:compile-test-assets: rules: @@ -434,26 +444,26 @@ - <<: *if-merge-request changes: *frontend-patterns -.frontend:rules:ee-mr-and-master-only: +.frontend:rules:ee-mr-and-default-branch-only: rules: - <<: *if-not-ee when: never - <<: *if-merge-request changes: *code-backstage-patterns when: always - - <<: *if-master-refs + - <<: *if-default-branch-refs changes: *code-backstage-patterns .frontend:rules:qa-frontend-node: rules: - - <<: *if-master-refs + - <<: *if-default-branch-refs changes: *frontend-dependency-patterns - <<: *if-merge-request changes: *frontend-dependency-patterns .frontend:rules:qa-frontend-node-latest: rules: - - <<: *if-master-refs + - <<: *if-default-branch-refs changes: *frontend-dependency-patterns allow_failure: true - <<: *if-merge-request @@ -464,8 +474,8 @@ rules: - <<: *if-not-canonical-namespace when: never - - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main")' - changes: *frontend-patterns + - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' + changes: *frontend-build-patterns allow_failure: true ################ @@ -484,7 +494,7 @@ rules: - <<: *if-not-ee when: never - - <<: *if-master-schedule-2-hourly + - <<: *if-default-branch-schedule-2-hourly ############ # QA rules # @@ -553,6 +563,7 @@ when: never - <<: *if-merge-request changes: *db-patterns + when: manual .rails:rules:ee-and-foss-unit: rules: @@ -824,14 +835,14 @@ - changes: *db-library-patterns - <<: *if-merge-request-title-run-all-rspec -.rails:rules:ee-mr-and-master-only: +.rails:rules:ee-mr-and-default-branch-only: rules: - <<: *if-not-ee when: never - <<: *if-merge-request-title-run-all-rspec - <<: *if-merge-request changes: *code-backstage-patterns - - <<: *if-master-refs + - <<: *if-default-branch-refs changes: *code-backstage-patterns .rails:rules:detect-tests: @@ -878,16 +889,11 @@ changes: *code-backstage-patterns when: on_failure -.rails:rules:downtime_check: - rules: - - <<: *if-merge-request - changes: *code-backstage-patterns - .rails:rules:deprecations: rules: - <<: *if-not-ee when: never - - <<: *if-master-schedule-nightly + - <<: *if-default-branch-schedule-nightly - <<: *if-merge-request-title-run-all-rspec .rails:rules:rspec-coverage: @@ -897,7 +903,7 @@ - <<: *if-merge-request changes: *code-backstage-patterns when: always - - <<: *if-master-schedule-2-hourly + - <<: *if-default-branch-schedule-2-hourly - <<: *if-merge-request-title-run-all-rspec when: always @@ -905,24 +911,34 @@ rules: - <<: *if-not-ee when: never - - <<: *if-master-schedule-2-hourly + - <<: *if-default-branch-schedule-2-hourly allow_failure: true - <<: *if-merge-request-title-run-all-rspec -.rails:rules:master-schedule-nightly--code-backstage: +.rails:rules:default-branch-schedule-nightly--code-backstage: rules: - - <<: *if-master-schedule-nightly + - <<: *if-default-branch-schedule-nightly - <<: *if-merge-request changes: [".gitlab/ci/rails.gitlab-ci.yml"] -.rails:rules:master-schedule-nightly--code-backstage-ee-only: +.rails:rules:default-branch-schedule-nightly--code-backstage-ee-only: rules: - <<: *if-not-ee when: never - - <<: *if-master-schedule-nightly + - <<: *if-default-branch-schedule-nightly - <<: *if-merge-request changes: [".gitlab/ci/rails.gitlab-ci.yml"] +####################### +# Vendored gems rules # +####################### + +.vendor:rules:mail-smtp_pool: + rules: + - <<: *if-merge-request + changes: ["vendor/gems/mail-smtp_pool/**/*"] + - <<: *if-merge-request-title-run-all-rspec + ################## # Releases rules # ################## @@ -945,7 +961,7 @@ rules: - if: '$CODE_QUALITY_DISABLED' when: never - # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 + # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-patterns allow_failure: true @@ -954,7 +970,7 @@ rules: - if: '$SAST_DISABLED || $GITLAB_FEATURES !~ /\bsast\b/' when: never - # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 + # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns allow_failure: true @@ -963,7 +979,7 @@ rules: - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/' when: never - # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 + # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns allow_failure: true @@ -984,7 +1000,7 @@ rules: - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' when: never - - <<: *if-master-schedule-nightly + - <<: *if-default-branch-schedule-nightly allow_failure: true .reports:rules:license_scanning: @@ -1007,9 +1023,12 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - <<: *if-dot-com-gitlab-org-merge-request - changes: *code-qa-patterns + changes: *code-patterns when: manual allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns + allow_failure: true - <<: *if-dot-com-gitlab-org-schedule .review:rules:review-deploy: @@ -1022,9 +1041,12 @@ changes: *frontend-patterns allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request - changes: *code-qa-patterns + changes: *code-patterns when: manual allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns + allow_failure: true - <<: *if-dot-com-gitlab-org-schedule allow_failure: true @@ -1067,14 +1089,17 @@ when: manual allow_failure: true -.review:rules:mr-only-manual: +.review:rules:review-qa-all: rules: - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-merge-request - changes: *code-qa-patterns + changes: *code-patterns when: manual allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns + allow_failure: true .review:rules:review-cleanup: rules: @@ -1100,7 +1125,7 @@ .review:rules:danger: rules: - - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID' + - if: '$CI_MERGE_REQUEST_IID' ############### # Setup rules # @@ -1109,13 +1134,13 @@ rules: - <<: *if-not-canonical-namespace when: never - - <<: *if-master-or-tag + - <<: *if-default-branch-or-tag changes: *code-backstage-qa-patterns when: on_success .setup:rules:dont-interrupt-me: rules: - - <<: *if-master-or-tag + - <<: *if-default-branch-or-tag allow_failure: true - <<: *if-auto-deploy-branches allow_failure: true diff --git a/.gitlab/ci/untamper-my-lockfile.yml b/.gitlab/ci/untamper-my-lockfile.yml new file mode 100644 index 00000000000..54ba160f1bd --- /dev/null +++ b/.gitlab/ci/untamper-my-lockfile.yml @@ -0,0 +1,26 @@ +untamper-my-lockfile: + image: registry.gitlab.com/gitlab-org/frontend/untamper-my-lockfile:main + stage: test + needs: [] + before_script: [] + after_script: [] + cache: {} + retry: 1 + script: + - untamper-my-lockfile --lockfile yarn.lock + rules: + # Create a pipeline if the branch is named 'add-untamper-my-lockfile' in + # order to have an integration check added in the MR that introduces it + - if: $CI_COMMIT_REF_NAME == "add-untamper-my-lockfile" + # Create a pipeline if there are changes in yarn.lock _and_ we are in a + # merge request _or_ branch pipeline. + # + # This ensures that the pipeline isn't run in scheduled jobs for example + # + # Also our best effort to support both branch and MR pipelines. In certain + # projects this might trigger _two_ pipelines. These projects can be fixed + # by adding proper workflow:rules + # https://docs.gitlab.com/ee/ci/yaml/#workflowrules + - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH + changes: + - yarn.lock diff --git a/.gitlab/ci/vendored-gems.gitlab-ci.yml b/.gitlab/ci/vendored-gems.gitlab-ci.yml new file mode 100644 index 00000000000..a39c4307c13 --- /dev/null +++ b/.gitlab/ci/vendored-gems.gitlab-ci.yml @@ -0,0 +1,7 @@ +vendor mail-smtp_pool: + extends: + - .vendor:rules:mail-smtp_pool + needs: [] + trigger: + include: vendor/gems/mail-smtp_pool/.gitlab-ci.yml + strategy: depend diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index 8361d20d2b7..ba4523f3bf7 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -1,6 +1,6 @@ workhorse:verify: extends: .workhorse:rules:workhorse - image: ${GITLAB_DEPENDENCY_PROXY}golang:1.15 + image: ${GITLAB_DEPENDENCY_PROXY}golang:1.16 stage: test needs: [] script: @@ -23,14 +23,10 @@ workhorse:verify: - apt-get update && apt-get -y install libimage-exiftool-perl - make -C workhorse test -workhorse:test using go 1.13: - extends: .workhorse:test - image: ${GITLAB_DEPENDENCY_PROXY}golang:1.13 - -workhorse:test using go 1.14: - extends: .workhorse:test - image: ${GITLAB_DEPENDENCY_PROXY}golang:1.14 - workhorse:test using go 1.15: extends: .workhorse:test image: ${GITLAB_DEPENDENCY_PROXY}golang:1.15 + +workhorse:test using go 1.16: + extends: .workhorse:test + image: ${GITLAB_DEPENDENCY_PROXY}golang:1.16 diff --git a/.gitlab/issue_templates/Experiment Successful Cleanup.md b/.gitlab/issue_templates/Experiment Successful Cleanup.md index afe4793cdfc..42f26342342 100644 --- a/.gitlab/issue_templates/Experiment Successful Cleanup.md +++ b/.gitlab/issue_templates/Experiment Successful Cleanup.md @@ -15,5 +15,6 @@ The changes need to become an official part of the product. - [ ] Optional: Migrate experiment to a default enabled [feature flag](https://docs.gitlab.com/ee/development/feature_flags) for one milestone and add a changelog. Converting to a feature flag can be skipped at the ICs discretion if risk is deemed low with consideration to both SaaS and (if applicable) self managed - [ ] In the next milestone, [remove the feature flag](https://docs.gitlab.com/ee/development/feature_flags/controls.html#cleaning-up) if applicable - [ ] After the flag removal is deployed, [clean up the feature/experiment feature flags](https://docs.gitlab.com/ee/development/feature_flags/controls.html#cleaning-up) by running chatops command in `#production` channel +- [ ] Ensure the corresponding [Experiment Tracking](https://gitlab.com/groups/gitlab-org/-/boards/1352542?label_name[]=devops%3A%3Agrowth&label_name[]=growth%20experiment&label_name[]=experiment%20tracking) issue is updated /label ~"feature" ~"feature::maintenance" ~"workflow::scheduling" ~"growth experiment" ~"feature flag" diff --git a/.gitlab/issue_templates/Experimentation.md b/.gitlab/issue_templates/Experimentation.md index f84c4305c2c..ba7839fb941 100644 --- a/.gitlab/issue_templates/Experimentation.md +++ b/.gitlab/issue_templates/Experimentation.md @@ -18,7 +18,7 @@ # Tracking Details - [json schema](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_experiment/jsonschema/0-3-0) used in `gitlab-experiment` tracking. -- see [taxonomy](https://docs.gitlab.com/ee/development/snowplow.html#structured-event-taxonomy) for a guide. +- see [taxonomy](https://docs.gitlab.com/ee/development/snowplow/index.html#structured-event-taxonomy) for a guide. | activity | category | action | label | context | property | value | | -------- | -------- | ------ | ----- | ------- | -------- | ----- | diff --git a/.gitlab/issue_templates/Feature Flag Roll Out.md b/.gitlab/issue_templates/Feature Flag Roll Out.md index fe263b932ae..a67d0f4e31a 100644 --- a/.gitlab/issue_templates/Feature Flag Roll Out.md +++ b/.gitlab/issue_templates/Feature Flag Roll Out.md @@ -1,48 +1,107 @@ <!-- Title suggestion: [Feature flag] Enable description of feature --> -## What +## Feature -Remove the `:feature_name` feature flag ... +This feature uses the `:feature_name` feature flag! + +<!-- Short description of what the feature is about and link to relevant other issues. --> +- [Issue Name](ISSUE LINK) ## Owners - Team: NAME_OF_TEAM - Most appropriate slack channel to reach out to: `#g_TEAM_NAME` - Best individual to reach out to: NAME +- PM: NAME -## Expectations +## Stakeholders -### What are we expecting to happen? - -### What might happen if this goes wrong? +<!-- +Are there any other stages or teams involved that need to be kept in the loop? -### What can we monitor to detect problems with this? +- Name of a PM +- The Support Team +- The Delivery Team +--> -<!-- Which dashboards from https://dashboards.gitlab.net are most relevant? Sentry errors reports can also be useful to review --> +## The Rollout Plan +- Partial Rollout on GitLab.com with beta groups +- Rollout on GitLab.com for a certain period (How long) +- Percentage Rollout on GitLab.com +- Rollout Feature for everyone as soon as it's ready -## Beta groups/projects +<!-- Which dashboards from https://dashboards.gitlab.net are most relevant? Sentry errors reports can also be useful to review --> -If applicable, any groups/projects that are happy to have this feature turned on early. Some organizations may wish to test big changes they are interested in with a small subset of users ahead of time for example. +**Beta Groups/Projects:** +<!-- If applicable, any groups/projects that are happy to have this feature turned on early. Some organizations may wish to test big changes they are interested in with a small subset of users ahead of time for example. --> - `gitlab-org/gitlab` project - `gitlab-org`/`gitlab-com` groups - ... -## Roll Out Steps +## Expectations + +### What are we expecting to happen? + +<!-- Describe the expected outcome when rolling out this feature --> + +### What might happen if this goes wrong? + +<!-- Should the feature flag be turned off? Any MRs that need to be rolled back? Communication that needs to happen? What are some things you can think of that could go wrong - data loss or broken pages? --> + +### What can we monitor to detect problems with this? + +<!-- Which dashboards from https://dashboards.gitlab.net are most relevant? --> + +## Rollout Timeline + +<!-- Please check which steps are needed and remove those which don't apply --> + +**Initial Rollout** + +*Preparation Phase* - [ ] Enable on staging (`/chatops run feature set feature_name true --staging`) + - [ ] Test on staging -- [ ] Ensure that documentation has been updated -- [ ] Enable on GitLab.com for individual groups/projects listed above and verify behaviour (`/chatops run feature set --project=gitlab-org/gitlab feature_name true`) -- [ ] Coordinate a time to enable the flag with the SRE oncall and release managers - - In `#production` mention `@sre-oncall` and `@release-managers`. Once an SRE on call and Release Manager on call confirm, you can proceed with the rollout + +- [ ] Ensure that documentation has been updated ([More info](https://docs.gitlab.com/ee/development/documentation/feature_flags.html#features-that-became-enabled-by-default)) + - [ ] Announce on the issue an estimated time this will be enabled on GitLab.com -- [ ] Enable on GitLab.com by running chatops command in `#production` (`/chatops run feature set feature_name true`) -- [ ] Cross post chatops Slack command to `#support_gitlab-com` ([more guidance when this is necessary in the dev docs](https://docs.gitlab.com/ee/development/feature_flags/controls.html#where-to-run-commands)) and in your team channel + +*Partial Rollout Phase* +- [ ] Enable on GitLab.com for individual groups/projects listed above and verify behaviour (`/chatops run feature set --project=gitlab-org/gitlab feature_name true`) + +- [ ] Verify behaviour (See Beta Groups) and add details with screenshots as a comment on this issue + +- [ ] If it is possible to perform an incremental rollout, this should be preferred. Proposed increments are: `10%`, `50%`, `100%`. Proposed minimum time between increments is 15 minutes. + - When setting percentages, make sure that the feature works correctly between feature checks. See https://gitlab.com/gitlab-org/gitlab/-/issues/327117 for more information + - For actor-based rollout: `/chatops run feature set feature_name 10 --actors` + - For time-based rollout: `/chatops run feature set feature_name 10` + +- [ ] Make the feature flag enabled by default i.e. Change `default_enabled` to `true` + +- [ ] Cross post chatops slack command to `#support_gitlab-com` ([more guidance when this is necessary in the dev docs](https://docs.gitlab.com/ee/development/feature_flags/controls.html#where-to-run-commands)) and in your team channel + + +**Cleanup** + +This is an __important__ phase, that should be either done in the next Milestone or as soon as possible. For the cleanup phase, please follow our documentation on how to [clean up the feature flag](https://docs.gitlab.com/ee/development/feature_flags/controls.html#cleaning-up). + +<!-- The checklist here is to keep track of it's status for stakeholders --> - [ ] Announce on the issue that the flag has been enabled -- [ ] Remove feature flag and add changelog entry. Ensure that the feature flag definition YAML file has been removed in the **same MR** that is removing the feature flag from the code -- [ ] After the flag removal is deployed, [clean up the feature flag](https://docs.gitlab.com/ee/development/feature_flags/controls.html#cleaning-up) by running chatops command in `#production` channel + +- [ ] Remove `:feature_name` feature flag + - [ ] Remove all references to the feature flag from the codebase + - [ ] Remove the YAML definitions for the feature from the repository + - [ ] Create a Changelog Entry + +- [ ] Clean up the feature flag from all environments by running this chatops command in `#production` channel `/chatops run feature delete some_feature`. + +**Final Step** + +- [ ] Close this rollout issue for the feature flag after the feature flag is removed from the codebase. ## Rollback Steps @@ -53,3 +112,4 @@ If applicable, any groups/projects that are happy to have this feature turned on ``` /label ~"feature flag" +/assign DRI diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index 2cdf2341c88..72ee11e6f96 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -113,3 +113,5 @@ Use the following resources to find the appropriate labels: /label ~devops:: ~group: ~Category: /label ~"GitLab Core"/~"GitLab Premium"/~"GitLab Ultimate" /label ~feature +/label ~documentation +/label ~direction diff --git a/.gitlab/issue_templates/Geo: Replicate a new Git repository type.md b/.gitlab/issue_templates/Geo: Replicate a new Git repository type.md new file mode 100644 index 00000000000..6b2d732f246 --- /dev/null +++ b/.gitlab/issue_templates/Geo: Replicate a new Git repository type.md @@ -0,0 +1,756 @@ +<!-- + +This template is based on a model named `CoolWidget`. + +To adapt this template, find and replace the following tokens: + +- `CoolWidget` +- `Cool Widget` +- `cool_widget` +- `coolWidget` + +If your Model's pluralized form is non-standard, i.e. it doesn't just end in `s`, then find and replace the following tokens *first*: + +- `CoolWidgets` +- `Cool Widgets` +- `cool_widgets` +- `coolWidgets` + +--> + +## Replicate Cool Widgets + +This issue is for implementing Geo replication and verification of Cool Widgets. + +For more background, see [Geo self-service framework](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/geo/framework.md). + +In order to implement and test this feature, you need to first [set up Geo locally](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/geo.md). + +There are three main sections below. It is a good idea to structure your merge requests this way as well: + +1. Modify database schemas to prepare to add Geo support for Cool Widgets +1. Implement Geo support of Cool Widgets behind a feature flag +1. Release Geo support of Cool Widgets + +It is also a good idea to first open a proof-of-concept merge request. It can be helpful for working out kinks and getting initial support and feedback from the Geo team. As an example, see the [Proof of Concept to replicate Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56423). + +### Modify database schemas to prepare to add Geo support for Cool Widgets + +You might do this section in its own merge request, but it is not required. + +#### Add the registry table to track replication and verification state + +Geo secondary sites have a [Geo tracking database](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/geo.md#tracking-database) independent of the main database. It is used to track the replication and verification state of all replicables. Every Model has a corresponding "registry" table in the Geo tracking database. + +- [ ] Create the migration file in `ee/db/geo/migrate`: + + ```shell + bin/rails generate geo_migration CreateCoolWidgetRegistry + ``` + +- [ ] Replace the contents of the migration file with the following. Note that we cannot add a foreign key constraint on `cool_widget_id` because the `cool_widgets` table is in a different database. The application code must handle logic such as propagating deletions. + + ```ruby + # frozen_string_literal: true + + class CreateCoolWidgetRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + unless table_exists?(:cool_widget_registry) + ActiveRecord::Base.transaction do + create_table :cool_widget_registry, id: :bigserial, force: :cascade do |t| + t.bigint :cool_widget_id, null: false + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :last_synced_at + t.datetime_with_timezone :retry_at + t.datetime_with_timezone :verified_at + t.datetime_with_timezone :verification_started_at + t.datetime_with_timezone :verification_retry_at + t.integer :state, default: 0, null: false, limit: 2 + t.integer :verification_state, default: 0, null: false, limit: 2 + t.integer :retry_count, default: 0, limit: 2, null: false + t.integer :verification_retry_count, default: 0, limit: 2, null: false + t.boolean :checksum_mismatch, default: false, null: false + t.boolean :force_to_redownload, default: false, null: false + t.boolean :missing_on_primary, default: false, null: false + t.binary :verification_checksum + t.binary :verification_checksum_mismatched + t.string :verification_failure, limit: 255 # rubocop:disable Migration/PreventStrings see https://gitlab.com/gitlab-org/gitlab/-/issues/323806 + t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings see https://gitlab.com/gitlab-org/gitlab/-/issues/323806 + + t.index :cool_widget_id, name: :index_cool_widget_registry_on_cool_widget_id, unique: true + t.index :retry_at + t.index :state + # To optimize performance of CoolWidgetRegistry.verification_failed_batch + t.index :verification_retry_at, name: :cool_widget_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + # To optimize performance of CoolWidgetRegistry.needs_verification_count + t.index :verification_state, name: :cool_widget_registry_needs_verification, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + # To optimize performance of CoolWidgetRegistry.verification_pending_batch + t.index :verified_at, name: :cool_widget_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + end + end + end + + def down + drop_table :cool_widget_registry + end + end + ``` + +- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). +- [ ] Run Geo tracking database migrations: + + ```shell + bin/rake geo:db:migrate + ``` + +- [ ] Be sure to commit the relevant changes in `ee/db/geo/schema.rb` + +### Add verification state fields on the Geo primary site + +The Geo primary site needs to checksum every replicable in order for secondaries to verify their own checksums. To do this, Geo requires fields on the Model. There are two ways to add the necessary verification state fields. If the table is large and wide, then it may be a good idea to add verification state fields to a separate table (Option 2). Consult a database expert if needed. + +#### Add verification state fields to the model table (Option 1) + +- [ ] Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration AddVerificationStateToCoolWidgets + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class AddVerificationStateToCoolWidgets < ActiveRecord::Migration[6.0] + def change + change_table(:cool_widgets) do |t| + t.integer :verification_state, default: 0, limit: 2, null: false + t.column :verification_started_at, :datetime_with_timezone + t.integer :verification_retry_count, limit: 2, null: false + t.column :verification_retry_at, :datetime_with_timezone + t.column :verified_at, :datetime_with_timezone + t.binary :verification_checksum, using: 'verification_checksum::bytea' + + t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns + end + end + end + ``` + +- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). +- [ ] If `cool_widgets` is a high-traffic table, follow [the database documentation to use `with_lock_retries`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/migration_style_guide.md#when-to-use-the-helper-method) +- [ ] Adding a `text` column also [requires](../database/strings_and_the_text_data_type.md#add-a-text-column-to-an-existing-table) setting a limit. Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration AddVerificationFailureLimitToCoolWidgets + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class AddVerificationFailureLimitToCoolWidgets < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + CONSTRAINT_NAME = 'cool_widget_verification_failure_text_limit' + + def up + add_text_limit :cool_widget, :verification_failure, 255, constraint_name: CONSTRAINT_NAME + end + + def down + remove_check_constraint(:cool_widget, CONSTRAINT_NAME) + end + end + ``` + +- [ ] Add indexes on verification fields to ensure verification can be performed efficiently. Some or all of these indexes can be omitted if the table is guaranteed to be small. Ask a database expert if you are considering omitting indexes. Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration AddVerificationIndexesToCoolWidgets + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class AddVerificationIndexesToCoolWidgets < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + VERIFICATION_STATE_INDEX_NAME = "index_cool_widgets_on_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_cool_widgets_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_cool_widgets_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_cool_widgets_needs_verification" + + disable_ddl_transaction! + + def up + add_concurrent_index :cool_widgets, :verification_state, name: VERIFICATION_STATE_INDEX_NAME + add_concurrent_index :cool_widgets, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + add_concurrent_index :cool_widgets, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + add_concurrent_index :cool_widgets, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :cool_widgets, VERIFICATION_STATE_INDEX_NAME + remove_concurrent_index_by_name :cool_widgets, PENDING_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :cool_widgets, FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :cool_widgets, NEEDS_VERIFICATION_INDEX_NAME + end + end + ``` + +- [ ] Run database migrations: + + ```shell + bin/rake db:migrate + ``` + +- [ ] Be sure to commit the relevant changes in `db/structure.sql` + +#### Add verification state fields to a separate table (Option 2) + +- [ ] Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration CreateCoolWidgetStates + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class CreateCoolWidgetStates < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + VERIFICATION_STATE_INDEX_NAME = "index_cool_widget_states_on_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_cool_widget_states_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_cool_widget_states_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_cool_widget_states_needs_verification" + + disable_ddl_transaction! + + def up + unless table_exists?(:cool_widget_states) + with_lock_retries do + create_table :cool_widget_states, id: false do |t| + t.references :cool_widget, primary_key: true, null: false, foreign_key: { on_delete: :cascade } + t.integer :verification_state, default: 0, limit: 2, null: false + t.column :verification_started_at, :datetime_with_timezone + t.datetime_with_timezone :verification_retry_at + t.datetime_with_timezone :verified_at + t.integer :verification_retry_count, limit: 2 + t.binary :verification_checksum, using: 'verification_checksum::bytea' + t.text :verification_failure + + t.index :verification_state, name: VERIFICATION_STATE_INDEX_NAME + t.index :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + t.index :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + t.index :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + end + end + + add_text_limit :cool_widget_states, :verification_failure, 255 + end + + def down + drop_table :cool_widget_states + end + end + ``` + +- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). +- [ ] Run database migrations: + + ```shell + bin/rake db:migrate + ``` + +- [ ] Be sure to commit the relevant changes in `db/structure.sql` + +That's all of the required database changes. + +### Implement Geo support of Cool Widgets behind a feature flag + +#### Step 1. Implement replication and verification + +- [ ] Include `Gitlab::Geo::ReplicableModel` in the `CoolWidget` class, and specify the Replicator class `with_replicator Geo::CoolWidgetReplicator`. + + Pay some attention to method `pool_repository`. Not every repository type uses repository pooling. As Geo prefers to use repository snapshotting, it can lead to data loss. Make sure to overwrite `pool_repository` so it returns nil for repositories that do not have pools. + + At this point the `CoolWidget` class should look like this: + + ```ruby + # frozen_string_literal: true + + class CoolWidget < ApplicationRecord + include ::Gitlab::Geo::ReplicableModel + include ::Gitlab::Geo::VerificationState + + with_replicator Geo::CoolWidgetReplicator + + mount_uploader :file, CoolWidgetUploader + + # Override the `all` default if not all records can be replicated. For an + # example of an existing Model that needs to do this, see + # `EE::MergeRequestDiff`. + # scope :available_replicables, -> { all } + + # @param primary_key_in [Range, CoolWidget] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation<CoolWidget>] everything that should be synced to this node, restricted by primary key + def self.replicables_for_current_secondary(primary_key_in) + # This issue template does not help you write this method. + # + # This method is called only on Geo secondary sites. It is called when + # we want to know which records to replicate. This is not easy to automate + # because for example: + # + # * The "selective sync" feature allows admins to choose which namespaces # to replicate, per secondary site. Most Models are scoped to a + # namespace, but the nature of the relationship to a namespace varies + # between Models. + # * The "selective sync" feature allows admins to choose which shards to + # replicate, per secondary site. Repositories are associated with + # shards. Most blob types are not, but Project Uploads are. + # * Remote stored replicables are not replicated, by default. But the + # setting `sync_object_storage` enables replication of remote stored + # replicables. + # + # Search the codebase for examples, and consult a Geo expert if needed. + end + + # Geo checks this method in FrameworkRepositorySyncService to avoid + # snapshotting repositories using object pools + def pool_repository + nil + end + + ... + end + ``` + +- [ ] Implement `CoolWidget.replicables_for_current_secondary` above. +- [ ] Ensure `CoolWidget.replicables_for_current_secondary` is well-tested. Search the codebase for `replicables_for_current_secondary` to find examples of parameterized table specs. You may need to add more `FactoryBot` traits. +- [ ] If you are using a separate table `cool_widget_states` to track verification state on the Geo primary site, then: + - [ ] Do not include `::Gitlab::Geo::VerificationState` on the `CoolWidget` class. + - [ ] Add the following lines to the `cool_widget_state.rb` model: + + ```ruby + class CoolWidgetState < ApplicationRecord + ... + self.primary_key = :cool_widget_id + + include ::Gitlab::Geo::VerificationState + + belongs_to :cool_widget, inverse_of: :cool_widget_state + ... + end + ``` + + - [ ] Add the following lines to the `cool_widget` model: + + ```ruby + class CoolWidget < ApplicationRecord + ... + has_one :cool_widget_state, inverse_of: :cool_widget + + delegate :verification_retry_at, :verification_retry_at=, + :verified_at, :verified_at=, + :verification_checksum, :verification_checksum=, + :verification_failure, :verification_failure=, + :verification_retry_count, :verification_retry_count=, + to: :cool_widget_state + ... + end + ``` + +- [ ] Create `ee/app/replicators/geo/cool_widget_replicator.rb`. Implement the `#repository` method which should return a `<Repository>` instance, and implement the class method `.model` to return the `CoolWidget` class: + + ```ruby + # frozen_string_literal: true + + module Geo + class CoolWidgetReplicator < Gitlab::Geo::Replicator + include ::Geo::RepositoryReplicatorStrategy + + def self.model + ::CoolWidget + end + + def repository + model_record.repository + end + + def self.git_access_class + ::Gitlab::GitAccessCoolWidget + end + + # The feature flag follows the format `geo_#{replicable_name}_replication`, + # so here it would be `geo_cool_widget_replication` + def self.replication_enabled_by_default? + false + end + + override :verification_feature_flag_enabled? + def self.verification_feature_flag_enabled? + # We are adding verification at the same time as replication, so we + # don't need to toggle verification separately from replication. When + # the replication feature flag is off, then verification is also off + # (see `VerifiableReplicator.verification_enabled?`) + true + end + + end + end + ``` + +- [ ] Make sure Geo push events are created. Usually it needs some change in the `app/workers/post_receive.rb` file. Example: + + ```ruby + def replicate_cool_widget_changes(cool_widget) + if ::Gitlab::Geo.primary? + cool_widget.replicator.handle_after_update if cool_widget + end + end + ``` + + See `app/workers/post_receive.rb` for more examples. + +- [ ] Make sure the repository removal is also handled. You may need to add something like the following in the destroy service of the repository: + + ```ruby + cool_widget.replicator.handle_after_destroy if cool_widget.repository + ``` + +- [ ] Make sure a Geo secondary site can request and download Cool Widgets on the Geo primary site. You may need to make some changes to `Gitlab::GitAccessCoolWidget`. For example, see [this change for Group-level Wikis](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54914/diffs?commit_id=0f2b36f66697b4addbc69bd377ee2818f648dd33). +- [ ] Generate the feature flag definition file by running the feature flag command and following the command prompts: + + ```shell + bin/feature-flag --ee geo_cool_widget_replication --type development --group 'group::geo' + ``` + +- [ ] Add this replicator class to the method `replicator_classes` in + `ee/lib/gitlab/geo.rb`: + + ```ruby + REPLICATOR_CLASSES = [ + ::Geo::PackageFileReplicator, + ::Geo::CoolWidgetReplicator + ] + end + ``` + +- [ ] Create `ee/spec/replicators/geo/cool_widget_replicator_spec.rb` and perform the necessary setup to define the `model_record` variable for the shared examples: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Geo::CoolWidgetReplicator do + let(:model_record) { build(:cool_widget) } + + include_examples 'a repository replicator' + include_examples 'a verifiable replicator' + end + ``` + +- [ ] Create `ee/app/models/geo/cool_widget_registry.rb`: + + ```ruby + # frozen_string_literal: true + + class Geo::CoolWidgetRegistry < Geo::BaseRegistry + include ::Geo::ReplicableRegistry + include ::Geo::VerifiableRegistry + + MODEL_CLASS = ::CoolWidget + MODEL_FOREIGN_KEY = :cool_widget_id + + belongs_to :cool_widget, class_name: 'CoolWidget' + end + ``` + +- [ ] Update `REGISTRY_CLASSES` in `ee/app/workers/geo/secondary/registry_consistency_worker.rb`. +- [ ] Update `def model_class_factory_name` in `ee/spec/services/geo/registry_consistency_service_spec.rb`. +- [ ] Update `it 'creates missing registries for each registry class'` in `ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb`. +- [ ] Add `cool_widget_registry` to `ActiveSupport::Inflector.inflections` in `config/initializers_before_autoloader/000_inflections.rb`. +- [ ] Create `ee/spec/factories/geo/cool_widget_registry.rb`: + + ```ruby + # frozen_string_literal: true + + FactoryBot.define do + factory :geo_cool_widget_registry, class: 'Geo::CoolWidgetRegistry' do + cool_widget + state { Geo::CoolWidgetRegistry.state_value(:pending) } + + trait :synced do + state { Geo::CoolWidgetRegistry.state_value(:synced) } + last_synced_at { 5.days.ago } + end + + trait :failed do + state { Geo::CoolWidgetRegistry.state_value(:failed) } + last_synced_at { 1.day.ago } + retry_count { 2 } + last_sync_failure { 'Random error' } + end + + trait :started do + state { Geo::CoolWidgetRegistry.state_value(:started) } + last_synced_at { 1.day.ago } + retry_count { 0 } + end + end + end + ``` + +- [ ] Create `ee/spec/models/geo/cool_widget_registry_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Geo::CoolWidgetRegistry, :geo, type: :model do + let_it_be(:registry) { create(:geo_cool_widget_registry) } + + specify 'factory is valid' do + expect(registry).to be_valid + end + + include_examples 'a Geo framework registry' + include_examples 'a Geo verifiable registry' + end + ``` + +#### Step 2. Implement metrics gathering + +Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` for display in the UI, and sent to Prometheus: + +- [ ] Add the following fields to Geo Node Status example responses in `doc/api/geo_nodes.md`: + - `cool_widgets_count` + - `cool_widgets_checksum_total_count` + - `cool_widgets_checksummed_count` + - `cool_widgets_checksum_failed_count` + - `cool_widgets_synced_count` + - `cool_widgets_failed_count` + - `cool_widgets_registry_count` + - `cool_widgets_verification_total_count` + - `cool_widgets_verified_count` + - `cool_widgets_verification_failed_count` + - `cool_widgets_synced_in_percentage` + - `cool_widgets_verified_in_percentage` +- [ ] Add the same fields to `GET /geo_nodes/status` example response in + `ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json`. +- [ ] Add the following fields to the `Sidekiq metrics` table in `doc/administration/monitoring/prometheus/gitlab_metrics.md`: + - `geo_cool_widgets` + - `geo_cool_widgets_checksum_total` + - `geo_cool_widgets_checksummed` + - `geo_cool_widgets_checksum_failed` + - `geo_cool_widgets_synced` + - `geo_cool_widgets_failed` + - `geo_cool_widgets_registry` + - `geo_cool_widgets_verification_total` + - `geo_cool_widgets_verified` + - `geo_cool_widgets_verification_failed` +- [ ] Add the following to the parameterized table in the `context 'Replicator stats' do` block in `ee/spec/models/geo_node_status_spec.rb`: + + ```ruby + Geo::CoolWidgetReplicator | :cool_widget | :geo_cool_widget_registry + ``` + +- [ ] Add the following to `spec/factories/cool_widgets.rb`: + + ```ruby + trait(:verification_succeeded) do + with_file + verification_checksum { 'abc' } + verification_state { CoolWidget.verification_state_value(:verification_succeeded) } + end + + trait(:verification_failed) do + with_file + verification_failure { 'Could not calculate the checksum' } + verification_state { CoolWidget.verification_state_value(:verification_failed) } + end + ``` + +- [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. + +Cool Widget replication and verification metrics should now be available in the API, the `Admin > Geo > Nodes` view, and Prometheus. + +#### Step 3. Implement the GraphQL API + +The GraphQL API is used by `Admin > Geo > Replication Details` views, and is directly queryable by administrators. + +- [ ] Add a new field to `GeoNodeType` in `ee/app/graphql/types/geo/geo_node_type.rb`: + + ```ruby + field :cool_widget_registries, ::Types::Geo::CoolWidgetRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::CoolWidgetRegistriesResolver, + description: 'Find Cool Widget registries on this Geo node', + feature_flag: :geo_cool_widget_replication + ``` + +- [ ] Add the new `cool_widget_registries` field name to the `expected_fields` array in `ee/spec/graphql/types/geo/geo_node_type_spec.rb`. +- [ ] Create `ee/app/graphql/resolvers/geo/cool_widget_registries_resolver.rb`: + + ```ruby + # frozen_string_literal: true + + module Resolvers + module Geo + class CoolWidgetRegistriesResolver < BaseResolver + type ::Types::Geo::GeoNodeType.connection_type, null: true + + include RegistriesResolver + end + end + end + ``` + +- [ ] Create `ee/spec/graphql/resolvers/geo/cool_widget_registries_resolver_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Resolvers::Geo::CoolWidgetRegistriesResolver do + it_behaves_like 'a Geo registries resolver', :geo_cool_widget_registry + end + ``` + +- [ ] Create `ee/app/finders/geo/cool_widget_registry_finder.rb`: + + ```ruby + # frozen_string_literal: true + + module Geo + class CoolWidgetRegistryFinder + include FrameworkRegistryFinder + end + end + ``` + +- [ ] Create `ee/spec/finders/geo/cool_widget_registry_finder_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Geo::CoolWidgetRegistryFinder do + it_behaves_like 'a framework registry finder', :geo_cool_widget_registry + end + ``` + +- [ ] Create `ee/app/graphql/types/geo/cool_widget_registry_type.rb`: + + ```ruby + # frozen_string_literal: true + + module Types + module Geo + # rubocop:disable Graphql/AuthorizeTypes because it is included + class CoolWidgetRegistryType < BaseObject + include ::Types::Geo::RegistryType + + graphql_name 'CoolWidgetRegistry' + description 'Represents the Geo replication and verification state of a cool_widget' + + field :cool_widget_id, GraphQL::ID_TYPE, null: false, description: 'ID of the Cool Widget' + end + end + end + ``` + +- [ ] Create `ee/spec/graphql/types/geo/cool_widget_registry_type_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe GitlabSchema.types['CoolWidgetRegistry'] do + it_behaves_like 'a Geo registry type' + + it 'has the expected fields (other than those included in RegistryType)' do + expected_fields = %i[cool_widget_id] + + expect(described_class).to have_graphql_fields(*expected_fields).at_least + end + end + ``` + +- [ ] Add integration tests for providing CoolWidget registry data to the frontend via the GraphQL API, by duplicating and modifying the following shared examples in `ee/spec/requests/api/graphql/geo/registries_spec.rb`: + + ```ruby + it_behaves_like 'gets registries for', { + field_name: 'coolWidgetRegistries', + registry_class_name: 'CoolWidgetRegistry', + registry_factory: :geo_cool_widget_registry, + registry_foreign_key_field_name: 'coolWidgetId' + } + ``` + +- [ ] Update the GraphQL reference documentation: + + ```shell + bundle exec rake gitlab:graphql:compile_docs + ``` + +Individual Cool Widget replication and verification data should now be available via the GraphQL API. + +### Release Geo support of Cool Widgets + +- [ ] In the rollout issue you created when creating the feature flag, modify the Roll Out Steps: + - [ ] Cross out any steps related to testing on production GitLab.com, because Geo is not running on production GitLab.com at the moment. + - [ ] Add a step to `Test replication and verification of Cool Widgets on a non-GDK-deployment. For example, using GitLab Environment Toolkit`. + - [ ] Add a step to `Ping the Geo PM and EM to coordinate testing`. For example, you might add steps to generate Cool Widgets, and then a Geo engineer may take it from there. +- [ ] In `ee/config/feature_flags/development/geo_cool_widget_replication.yml`, set `default_enabled: true` + +- [ ] In `ee/app/replicators/geo/cool_widget_replicator.rb`, delete the `self.replication_enabled_by_default?` method: + + ```ruby + module Geo + class CoolWidgetReplicator < Gitlab::Geo::Replicator + ... + + # REMOVE THIS METHOD + def self.replication_enabled_by_default? + false + end + # REMOVE THIS METHOD + + ... + end + end + ``` + +- [ ] In `ee/app/graphql/types/geo/geo_node_type.rb`, remove the `feature_flag` option for the released type: + + ```ruby + field :cool_widget_registries, ::Types::Geo::CoolWidgetRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::CoolWidgetRegistriesResolver, + description: 'Find Cool Widget registries on this Geo node', + feature_flag: :geo_cool_widget_replication # REMOVE THIS LINE + ``` + +- [ ] Add a row for Cool Widgets to the `Data types` table in [Geo data types support](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/geo/replication/datatypes.md#data-types) +- [ ] Add a row for Cool Widgets to the `Limitations on replication/verification` table in [Geo data types support](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/geo/replication/datatypes.md#limitations-on-replicationverification). If the row already exists, then update it to show that Replication and Verification is released in the current version. diff --git a/.gitlab/issue_templates/Geo: Replicate a new blob type.md b/.gitlab/issue_templates/Geo: Replicate a new blob type.md new file mode 100644 index 00000000000..12fe6a6f5bb --- /dev/null +++ b/.gitlab/issue_templates/Geo: Replicate a new blob type.md @@ -0,0 +1,722 @@ +<!-- + +This template is based on a model named `CoolWidget`. + +To adapt this template, find and replace the following tokens: + +- `CoolWidget` +- `Cool Widget` +- `cool_widget` +- `coolWidget` + +If your Model's pluralized form is non-standard, i.e. it doesn't just end in `s`, find and replace the following tokens *first*: + +- `CoolWidgets` +- `Cool Widgets` +- `cool_widgets` +- `coolWidgets` + +--> + +## Replicate Cool Widgets + +This issue is for implementing Geo replication and verification of Cool Widgets. + +For more background, see [Geo self-service framework](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/geo/framework.md). + +In order to implement and test this feature, you need to first [set up Geo locally](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/geo.md). + +There are three main sections below. It is a good idea to structure your merge requests this way as well: + +1. Modify database schemas to prepare to add Geo support for Cool Widgets +1. Implement Geo support of Cool Widgets behind a feature flag +1. Release Geo support of Cool Widgets + +It is also a good idea to first open a proof-of-concept merge request. It can be helpful for working out kinks and getting initial support and feedback from the Geo team. As an example, see the [Proof of Concept to replicate Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56423). + +### Modify database schemas to prepare to add Geo support for Cool Widgets + +You might do this section in its own merge request, but it is not required. + +#### Add the registry table to track replication and verification state + +Geo secondary sites have a [Geo tracking database](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/geo.md#tracking-database) independent of the main database. It is used to track the replication and verification state of all replicables. Every Model has a corresponding "registry" table in the Geo tracking database. + +- [ ] Create the migration file in `ee/db/geo/migrate`: + + ```shell + bin/rails generate geo_migration CreateCoolWidgetRegistry + ``` + +- [ ] Replace the contents of the migration file with the following. Note that we cannot add a foreign key constraint on `cool_widget_id` because the `cool_widgets` table is in a different database. The application code must handle logic such as propagating deletions. + + ```ruby + # frozen_string_literal: true + + class CreateCoolWidgetRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + unless table_exists?(:cool_widget_registry) + ActiveRecord::Base.transaction do + create_table :cool_widget_registry, id: :bigserial, force: :cascade do |t| + t.bigint :cool_widget_id, null: false + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :last_synced_at + t.datetime_with_timezone :retry_at + t.datetime_with_timezone :verified_at + t.datetime_with_timezone :verification_started_at + t.datetime_with_timezone :verification_retry_at + t.integer :state, default: 0, null: false, limit: 2 + t.integer :verification_state, default: 0, null: false, limit: 2 + t.integer :retry_count, default: 0, limit: 2, null: false + t.integer :verification_retry_count, default: 0, limit: 2, null: false + t.boolean :checksum_mismatch, default: false, null: false + t.binary :verification_checksum + t.binary :verification_checksum_mismatched + t.string :verification_failure, limit: 255 # rubocop:disable Migration/PreventStrings see https://gitlab.com/gitlab-org/gitlab/-/issues/323806 + t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings see https://gitlab.com/gitlab-org/gitlab/-/issues/323806 + + t.index :cool_widget_id, name: :index_cool_widget_registry_on_cool_widget_id, unique: true + t.index :retry_at + t.index :state + # To optimize performance of CoolWidgetRegistry.verification_failed_batch + t.index :verification_retry_at, name: :cool_widget_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + # To optimize performance of CoolWidgetRegistry.needs_verification_count + t.index :verification_state, name: :cool_widget_registry_needs_verification, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + # To optimize performance of CoolWidgetRegistry.verification_pending_batch + t.index :verified_at, name: :cool_widget_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + end + end + end + + def down + drop_table :cool_widget_registry + end + end + ``` + +- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). +- [ ] Run Geo tracking database migrations: + + ```shell + bin/rake geo:db:migrate + ``` + +- [ ] Be sure to commit the relevant changes in `ee/db/geo/schema.rb` + +### Add verification state fields on the Geo primary site + +The Geo primary site needs to checksum every replicable in order for secondaries to verify their own checksums. To do this, Geo requires fields on the Model. There are two ways to add the necessary verification state fields. If the table is large and wide, then it may be a good idea to add verification state fields to a separate table (Option 2). Consult a database expert if needed. + +#### Add verification state fields to the model table (Option 1) + +- [ ] Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration AddVerificationStateToCoolWidgets + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class AddVerificationStateToCoolWidgets < ActiveRecord::Migration[6.0] + def change + change_table(:cool_widgets) do |t| + t.integer :verification_state, default: 0, limit: 2, null: false + t.column :verification_started_at, :datetime_with_timezone + t.integer :verification_retry_count, limit: 2, null: false + t.column :verification_retry_at, :datetime_with_timezone + t.column :verified_at, :datetime_with_timezone + t.binary :verification_checksum, using: 'verification_checksum::bytea' + + t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns + end + end + end + ``` + +- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). +- [ ] If `cool_widgets` is a high-traffic table, follow [the database documentation to use `with_lock_retries`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/migration_style_guide.md#when-to-use-the-helper-method) +- [ ] Adding a `text` column also [requires](../database/strings_and_the_text_data_type.md#add-a-text-column-to-an-existing-table) setting a limit. Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration AddVerificationFailureLimitToCoolWidgets + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class AddVerificationFailureLimitToCoolWidgets < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + CONSTRAINT_NAME = 'cool_widget_verification_failure_text_limit' + + def up + add_text_limit :cool_widget, :verification_failure, 255, constraint_name: CONSTRAINT_NAME + end + + def down + remove_check_constraint(:cool_widget, CONSTRAINT_NAME) + end + end + ``` + +- [ ] Add indexes on verification fields to ensure verification can be performed efficiently. Some or all of these indexes can be omitted if the table is guaranteed to be small. Ask a database expert if you are considering omitting indexes. Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration AddVerificationIndexesToCoolWidgets + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class AddVerificationIndexesToCoolWidgets < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + VERIFICATION_STATE_INDEX_NAME = "index_cool_widgets_on_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_cool_widgets_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_cool_widgets_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_cool_widgets_needs_verification" + + disable_ddl_transaction! + + def up + add_concurrent_index :cool_widgets, :verification_state, name: VERIFICATION_STATE_INDEX_NAME + add_concurrent_index :cool_widgets, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + add_concurrent_index :cool_widgets, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + add_concurrent_index :cool_widgets, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :cool_widgets, VERIFICATION_STATE_INDEX_NAME + remove_concurrent_index_by_name :cool_widgets, PENDING_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :cool_widgets, FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :cool_widgets, NEEDS_VERIFICATION_INDEX_NAME + end + end + ``` + +- [ ] Run database migrations: + + ```shell + bin/rake db:migrate + ``` + +- [ ] Be sure to commit the relevant changes in `db/structure.sql` + +#### Add verification state fields to a separate table (Option 2) + +- [ ] Create the migration file in `db/migrate`: + + ```shell + bin/rails generate migration CreateCoolWidgetStates + ``` + +- [ ] Replace the contents of the migration file with: + + ```ruby + # frozen_string_literal: true + + class CreateCoolWidgetStates < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + VERIFICATION_STATE_INDEX_NAME = "index_cool_widget_states_on_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_cool_widget_states_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_cool_widget_states_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_cool_widget_states_needs_verification" + + disable_ddl_transaction! + + def up + unless table_exists?(:cool_widget_states) + with_lock_retries do + create_table :cool_widget_states, id: false do |t| + t.references :cool_widget, primary_key: true, null: false, foreign_key: { on_delete: :cascade } + t.integer :verification_state, default: 0, limit: 2, null: false + t.column :verification_started_at, :datetime_with_timezone + t.datetime_with_timezone :verification_retry_at + t.datetime_with_timezone :verified_at + t.integer :verification_retry_count, limit: 2 + t.binary :verification_checksum, using: 'verification_checksum::bytea' + t.text :verification_failure + + t.index :verification_state, name: VERIFICATION_STATE_INDEX_NAME + t.index :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + t.index :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + t.index :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + end + end + + add_text_limit :cool_widget_states, :verification_failure, 255 + end + + def down + drop_table :cool_widget_states + end + end + ``` + +- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). +- [ ] Run database migrations: + + ```shell + bin/rake db:migrate + ``` + +- [ ] Be sure to commit the relevant changes in `db/structure.sql` + +That's all of the required database changes. + +### Implement Geo support of Cool Widgets behind a feature flag + +#### Step 1. Implement replication and verification + +- [ ] Include `Gitlab::Geo::ReplicableModel` in the `CoolWidget` class, and specify the Replicator class `with_replicator Geo::CoolWidgetReplicator`. + + At this point the `CoolWidget` class should look like this: + + ```ruby + # frozen_string_literal: true + + class CoolWidget < ApplicationRecord + include ::Gitlab::Geo::ReplicableModel + include ::Gitlab::Geo::VerificationState + + with_replicator Geo::CoolWidgetReplicator + + mount_uploader :file, CoolWidgetUploader + + # Override the `all` default if not all records can be replicated. For an + # example of an existing Model that needs to do this, see + # `EE::MergeRequestDiff`. + # scope :available_replicables, -> { all } + + # @param primary_key_in [Range, CoolWidget] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation<CoolWidget>] everything that should be synced to this node, restricted by primary key + def self.replicables_for_current_secondary(primary_key_in) + # This issue template does not help you write this method. + # + # This method is called only on Geo secondary sites. It is called when + # we want to know which records to replicate. This is not easy to automate + # because for example: + # + # * The "selective sync" feature allows admins to choose which namespaces # to replicate, per secondary site. Most Models are scoped to a + # namespace, but the nature of the relationship to a namespace varies + # between Models. + # * The "selective sync" feature allows admins to choose which shards to + # replicate, per secondary site. Repositories are associated with + # shards. Most blob types are not, but Project Uploads are. + # * Remote stored replicables are not replicated, by default. But the + # setting `sync_object_storage` enables replication of remote stored + # replicables. + # + # Search the codebase for examples, and consult a Geo expert if needed. + end + ... + end + ``` + +- [ ] Implement `CoolWidget.replicables_for_current_secondary` above. +- [ ] Ensure `CoolWidget.replicables_for_current_secondary` is well-tested. Search the codebase for `replicables_for_current_secondary` to find examples of parameterized table specs. You may need to add more `FactoryBot` traits. +- [ ] If you are using a separate table `cool_widget_states` to track verification state on the Geo primary site, then: + - [ ] Do not include `::Gitlab::Geo::VerificationState` on the `CoolWidget` class. + - [ ] Add the following lines to the `cool_widget_state.rb` model: + + ```ruby + class CoolWidgetState < ApplicationRecord + ... + self.primary_key = :cool_widget_id + + include ::Gitlab::Geo::VerificationState + + belongs_to :cool_widget, inverse_of: :cool_widget_state + ... + end + ``` + + - [ ] Add the following lines to the `cool_widget` model: + + ```ruby + class CoolWidget < ApplicationRecord + ... + has_one :cool_widget_state, inverse_of: :cool_widget + + delegate :verification_retry_at, :verification_retry_at=, + :verified_at, :verified_at=, + :verification_checksum, :verification_checksum=, + :verification_failure, :verification_failure=, + :verification_retry_count, :verification_retry_count=, + to: :cool_widget_state + ... + end + ``` + +- [ ] Create `ee/app/replicators/geo/cool_widget_replicator.rb`. Implement the `#carrierwave_uploader` method which should return a `CarrierWave::Uploader`, and implement the class method `.model` to return the `CoolWidget` class: + + ```ruby + # frozen_string_literal: true + + module Geo + class CoolWidgetReplicator < Gitlab::Geo::Replicator + include ::Geo::BlobReplicatorStrategy + + def self.model + ::CoolWidget + end + + def carrierwave_uploader + model_record.file + end + + # The feature flag follows the format `geo_#{replicable_name}_replication`, + # so here it would be `geo_cool_widget_replication` + def self.replication_enabled_by_default? + false + end + + override :verification_feature_flag_enabled? + def self.verification_feature_flag_enabled? + # We are adding verification at the same time as replication, so we + # don't need to toggle verification separately from replication. When + # the replication feature flag is off, then verification is also off + # (see `VerifiableReplicator.verification_enabled?`) + true + end + + end + end + ``` + +- [ ] Generate the feature flag definition file by running the feature flag command and following the command prompts: + + ```shell + bin/feature-flag --ee geo_cool_widget_replication --type development --group 'group::geo' + ``` + +- [ ] Add this replicator class to the method `replicator_classes` in + `ee/lib/gitlab/geo.rb`: + + ```ruby + REPLICATOR_CLASSES = [ + ::Geo::PackageFileReplicator, + ::Geo::CoolWidgetReplicator + ] + end + ``` + +- [ ] Create `ee/spec/replicators/geo/cool_widget_replicator_spec.rb` and perform the necessary setup to define the `model_record` variable for the shared examples: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Geo::CoolWidgetReplicator do + let(:model_record) { build(:cool_widget) } + + include_examples 'a blob replicator' + include_examples 'a verifiable replicator' + end + ``` + +- [ ] Create `ee/app/models/geo/cool_widget_registry.rb`: + + ```ruby + # frozen_string_literal: true + + class Geo::CoolWidgetRegistry < Geo::BaseRegistry + include ::Geo::ReplicableRegistry + include ::Geo::VerifiableRegistry + + MODEL_CLASS = ::CoolWidget + MODEL_FOREIGN_KEY = :cool_widget_id + + belongs_to :cool_widget, class_name: 'CoolWidget' + end + ``` + +- [ ] Update `REGISTRY_CLASSES` in `ee/app/workers/geo/secondary/registry_consistency_worker.rb`. +- [ ] Update `def model_class_factory_name` in `ee/spec/services/geo/registry_consistency_service_spec.rb`. +- [ ] Update `it 'creates missing registries for each registry class'` in `ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb`. +- [ ] Add `cool_widget_registry` to `ActiveSupport::Inflector.inflections` in `config/initializers_before_autoloader/000_inflections.rb`. +- [ ] Create `ee/spec/factories/geo/cool_widget_registry.rb`: + + ```ruby + # frozen_string_literal: true + + FactoryBot.define do + factory :geo_cool_widget_registry, class: 'Geo::CoolWidgetRegistry' do + cool_widget + state { Geo::CoolWidgetRegistry.state_value(:pending) } + + trait :synced do + state { Geo::CoolWidgetRegistry.state_value(:synced) } + last_synced_at { 5.days.ago } + end + + trait :failed do + state { Geo::CoolWidgetRegistry.state_value(:failed) } + last_synced_at { 1.day.ago } + retry_count { 2 } + last_sync_failure { 'Random error' } + end + + trait :started do + state { Geo::CoolWidgetRegistry.state_value(:started) } + last_synced_at { 1.day.ago } + retry_count { 0 } + end + end + end + ``` + +- [ ] Create `ee/spec/models/geo/cool_widget_registry_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Geo::CoolWidgetRegistry, :geo, type: :model do + let_it_be(:registry) { create(:geo_cool_widget_registry) } + + specify 'factory is valid' do + expect(registry).to be_valid + end + + include_examples 'a Geo framework registry' + include_examples 'a Geo verifiable registry' + end + ``` + +#### Step 2. Implement metrics gathering + +Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` for display in the UI, and sent to Prometheus: + +- [ ] Add the following fields to Geo Node Status example responses in `doc/api/geo_nodes.md`: + - `cool_widgets_count` + - `cool_widgets_checksum_total_count` + - `cool_widgets_checksummed_count` + - `cool_widgets_checksum_failed_count` + - `cool_widgets_synced_count` + - `cool_widgets_failed_count` + - `cool_widgets_registry_count` + - `cool_widgets_verification_total_count` + - `cool_widgets_verified_count` + - `cool_widgets_verification_failed_count` + - `cool_widgets_synced_in_percentage` + - `cool_widgets_verified_in_percentage` +- [ ] Add the same fields to `GET /geo_nodes/status` example response in + `ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json`. +- [ ] Add the following fields to the `Sidekiq metrics` table in `doc/administration/monitoring/prometheus/gitlab_metrics.md`: + - `geo_cool_widgets` + - `geo_cool_widgets_checksum_total` + - `geo_cool_widgets_checksummed` + - `geo_cool_widgets_checksum_failed` + - `geo_cool_widgets_synced` + - `geo_cool_widgets_failed` + - `geo_cool_widgets_registry` + - `geo_cool_widgets_verification_total` + - `geo_cool_widgets_verified` + - `geo_cool_widgets_verification_failed` +- [ ] Add the following to the parameterized table in the `context 'Replicator stats' do` block in `ee/spec/models/geo_node_status_spec.rb`: + + ```ruby + Geo::CoolWidgetReplicator | :cool_widget | :geo_cool_widget_registry + ``` + +- [ ] Add the following to `spec/factories/cool_widgets.rb`: + + ```ruby + trait(:verification_succeeded) do + with_file + verification_checksum { 'abc' } + verification_state { CoolWidget.verification_state_value(:verification_succeeded) } + end + + trait(:verification_failed) do + with_file + verification_failure { 'Could not calculate the checksum' } + verification_state { CoolWidget.verification_state_value(:verification_failed) } + end + ``` + +- [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. + +Cool Widget replication and verification metrics should now be available in the API, the `Admin > Geo > Nodes` view, and Prometheus. + +#### Step 3. Implement the GraphQL API + +The GraphQL API is used by `Admin > Geo > Replication Details` views, and is directly queryable by administrators. + +- [ ] Add a new field to `GeoNodeType` in `ee/app/graphql/types/geo/geo_node_type.rb`: + + ```ruby + field :cool_widget_registries, ::Types::Geo::CoolWidgetRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::CoolWidgetRegistriesResolver, + description: 'Find Cool Widget registries on this Geo node', + feature_flag: :geo_cool_widget_replication + ``` + +- [ ] Add the new `cool_widget_registries` field name to the `expected_fields` array in `ee/spec/graphql/types/geo/geo_node_type_spec.rb`. +- [ ] Create `ee/app/graphql/resolvers/geo/cool_widget_registries_resolver.rb`: + + ```ruby + # frozen_string_literal: true + + module Resolvers + module Geo + class CoolWidgetRegistriesResolver < BaseResolver + type ::Types::Geo::GeoNodeType.connection_type, null: true + + include RegistriesResolver + end + end + end + ``` + +- [ ] Create `ee/spec/graphql/resolvers/geo/cool_widget_registries_resolver_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Resolvers::Geo::CoolWidgetRegistriesResolver do + it_behaves_like 'a Geo registries resolver', :geo_cool_widget_registry + end + ``` + +- [ ] Create `ee/app/finders/geo/cool_widget_registry_finder.rb`: + + ```ruby + # frozen_string_literal: true + + module Geo + class CoolWidgetRegistryFinder + include FrameworkRegistryFinder + end + end + ``` + +- [ ] Create `ee/spec/finders/geo/cool_widget_registry_finder_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe Geo::CoolWidgetRegistryFinder do + it_behaves_like 'a framework registry finder', :geo_cool_widget_registry + end + ``` + +- [ ] Create `ee/app/graphql/types/geo/cool_widget_registry_type.rb`: + + ```ruby + # frozen_string_literal: true + + module Types + module Geo + # rubocop:disable Graphql/AuthorizeTypes because it is included + class CoolWidgetRegistryType < BaseObject + include ::Types::Geo::RegistryType + + graphql_name 'CoolWidgetRegistry' + description 'Represents the Geo replication and verification state of a cool_widget' + + field :cool_widget_id, GraphQL::ID_TYPE, null: false, description: 'ID of the Cool Widget' + end + end + end + ``` + +- [ ] Create `ee/spec/graphql/types/geo/cool_widget_registry_type_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + RSpec.describe GitlabSchema.types['CoolWidgetRegistry'] do + it_behaves_like 'a Geo registry type' + + it 'has the expected fields (other than those included in RegistryType)' do + expected_fields = %i[cool_widget_id] + + expect(described_class).to have_graphql_fields(*expected_fields).at_least + end + end + ``` + +- [ ] Add integration tests for providing CoolWidget registry data to the frontend via the GraphQL API, by duplicating and modifying the following shared examples in `ee/spec/requests/api/graphql/geo/registries_spec.rb`: + + ```ruby + it_behaves_like 'gets registries for', { + field_name: 'coolWidgetRegistries', + registry_class_name: 'CoolWidgetRegistry', + registry_factory: :geo_cool_widget_registry, + registry_foreign_key_field_name: 'coolWidgetId' + } + ``` + +- [ ] Update the GraphQL reference documentation: + + ```shell + bundle exec rake gitlab:graphql:compile_docs + ``` + +Individual Cool Widget replication and verification data should now be available via the GraphQL API. + +### Release Geo support of Cool Widgets + +- [ ] In the rollout issue you created when creating the feature flag, modify the Roll Out Steps: + - [ ] Cross out any steps related to testing on production GitLab.com, because Geo is not running on production GitLab.com at the moment. + - [ ] Add a step to `Test replication and verification of Cool Widgets on a non-GDK-deployment. For example, using GitLab Environment Toolkit`. + - [ ] Add a step to `Ping the Geo PM and EM to coordinate testing`. For example, you might add steps to generate Cool Widgets, and then a Geo engineer may take it from there. +- [ ] In `ee/config/feature_flags/development/geo_cool_widget_replication.yml`, set `default_enabled: true` + +- [ ] In `ee/app/replicators/geo/cool_widget_replicator.rb`, delete the `self.replication_enabled_by_default?` method: + + ```ruby + module Geo + class CoolWidgetReplicator < Gitlab::Geo::Replicator + ... + + # REMOVE THIS METHOD + def self.replication_enabled_by_default? + false + end + # REMOVE THIS METHOD + + ... + end + end + ``` + +- [ ] In `ee/app/graphql/types/geo/geo_node_type.rb`, remove the `feature_flag` option for the released type: + + ```ruby + field :cool_widget_registries, ::Types::Geo::CoolWidgetRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::CoolWidgetRegistriesResolver, + description: 'Find Cool Widget registries on this Geo node', + feature_flag: :geo_cool_widget_replication # REMOVE THIS LINE + ``` + +- [ ] Add a row for Cool Widgets to the `Data types` table in [Geo data types support](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/geo/replication/datatypes.md#data-types) +- [ ] Add a row for Cool Widgets to the `Limitations on replication/verification` table in [Geo data types support](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/geo/replication/datatypes.md#limitations-on-replicationverification). If the row already exists, then update it to show that Replication and Verification is released in the current version. diff --git a/.gitlab/issue_templates/Implementation.md b/.gitlab/issue_templates/Implementation.md index 888c993766a..058834264b0 100644 --- a/.gitlab/issue_templates/Implementation.md +++ b/.gitlab/issue_templates/Implementation.md @@ -39,24 +39,25 @@ Add details for required items and delete others. <!-- Steps and the parts of the code that will need to get updated. The plan can also call-out responsibilities for other team members or teams. ---> + +e.g.: - [ ] ~frontend Step 1 - [ ] `@person` Step 1a - [ ] ~frontend Step 2 +--> + <!-- Workflow and other relevant labels -~"group::" ~"Category:" ~"GitLab Ultimate" ---> -/label ~"workflow::refinement" - -<!-- +# ~"group::" ~"Category:" ~"GitLab Ultimate" Other settings you might want to include when creating the issue. -/milestone %"Next 1-3 releases" -/assign @ -/epic & +# /assign @ +# /epic & --> + +/label ~"workflow::refinement" +/milestone %Backlog diff --git a/.gitlab/issue_templates/Lean Feature Proposal.md b/.gitlab/issue_templates/Lean Feature Proposal.md index fb9ac306f31..828d5161269 100644 --- a/.gitlab/issue_templates/Lean Feature Proposal.md +++ b/.gitlab/issue_templates/Lean Feature Proposal.md @@ -101,3 +101,6 @@ In which enterprise tier should this feature go? See https://about.gitlab.com/ha ### Is this a cross-stage feature? Communicate if this change will affect multiple Stage Groups or product areas. We recommend always start with the assumption that a feature request will have an impact into another Group. Loop in the most relevant PM and Product Designer from that Group to provide strategic support to help align the Group's broader plan and vision, as well as to avoid UX and technical debt. https://about.gitlab.com/handbook/product/#cross-stage-features --> + +/label ~documentation +/label ~direction diff --git a/.gitlab/issue_templates/Migrations.md b/.gitlab/issue_templates/OSS_Partner.md index 822722a0f71..d9c05026e7c 100644 --- a/.gitlab/issue_templates/Migrations.md +++ b/.gitlab/issue_templates/OSS_Partner.md @@ -1,10 +1,9 @@ -# Project Name | Migration Tracker -<!-- Please edit this header with your project / organization's name. --> +<!-- Please title your issue with the following format: "Project Name | Issue Tracker". --> ## Background <!-- -Please add information here about why you're planning on migrating. Include any initial announcements that have been made about the decision or status. +Please add information here about why your project is considering a migration to GitLab, or why it decided to do so. Include any initial announcements that have been / were made about the decision or status. --> ### Goals @@ -16,7 +15,7 @@ Please add information here about why you're planning on migrating. Include any <!-- Please complete as many items in this list as possible. If you're not sure yet, add "TBD" (To be Decided) or "Unknown" --> * **Timeline.** - - * **Product.** - GitLab Gold/Ultimate or Community Edition + * **Product.** - SaaS-Ultimate/Self-Managed-Ultimate or Community Edition * **Project's License.** What kind of OSI-approved license does your project use? ## Current Tooling and Replacements @@ -31,6 +30,8 @@ Please fill in the table to give an overview of your current tooling. Here's a d Here's an example of a replacements overview from one of the projects which migrated to GitLab: https://gitlab.com/gitlab-org/gitlab/-/issues/25657#gitlab-replacements +Consider deleting the table below if you are unable to expand upon your current tooling. + --> | Tool | Feature | GitLab feature | GitLab edition | @@ -63,5 +64,5 @@ Here is an example of what this list might look like once populated: https://git ------ -/label ~"Open Source" ~movingtogitlab -/cc @nuritzi +/label ~"Open Source Partners" +/cc @nuritzi @greg diff --git a/.gitlab/issue_templates/Productivity Improvement.md b/.gitlab/issue_templates/Productivity Improvement.md index 974f11f6da3..06692d3ede8 100644 --- a/.gitlab/issue_templates/Productivity Improvement.md +++ b/.gitlab/issue_templates/Productivity Improvement.md @@ -34,6 +34,6 @@ after the implementation is merged/deployed/released. - [ ] The solution improved the situation. - If yes, check this box and close the issue. Well done! :tada: - - Otherwise, create a new "Productivity Improvement" issue. You can re-use the description from this issue, but obviously another solution should be chosen this time. + - Otherwise, create a new "Productivity Improvement" issue. You can re-use the description from this issue, but another solution should be chosen this time. /label ~"Engineering Productivity" ~meta diff --git a/.gitlab/issue_templates/Query Performance Investigation.md b/.gitlab/issue_templates/Query Performance Investigation.md index ddd361e4f2f..354cdb1bfe1 100644 --- a/.gitlab/issue_templates/Query Performance Investigation.md +++ b/.gitlab/issue_templates/Query Performance Investigation.md @@ -10,6 +10,16 @@ As the name implies, the purpose of the template is to detail underperforming qu - [ ] Provide [priority and severity labels](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#availability) - [ ] If this requires immediate attention cc `@gitlab-org/database-team` and reach out in the #g_database slack channel +### SQL Statement + +```sql + +``` + +### Data from Elastic + +Instructions on collecting data from [PostgreSQL slow logs stored in Elasticsearch](https://gitlab.com/gitlab-com/runbooks/-/merge_requests/3361/diffs) + ### Requested Data points Please provide as many of these fields as possible when submitting a query performance report. @@ -20,7 +30,6 @@ Please provide as many of these fields as possible when submitting a query perfo - Database time relative to total database time - Source of calls (Sidekiq, WebAPI, etc) - Query ID -- SQL Statement - Query Plan - Query Example - Total number of calls (relative) diff --git a/.gitlab/issue_templates/Security developer workflow.md b/.gitlab/issue_templates/Security developer workflow.md index beb066cdfc4..25825fc8888 100644 --- a/.gitlab/issue_templates/Security developer workflow.md +++ b/.gitlab/issue_templates/Security developer workflow.md @@ -10,9 +10,10 @@ Set the title to: `Description of the original issue` - [ ] Read the [security process for developers] if you are not familiar with it. - Verify if the issue you're working on `gitlab-org/gitlab` is confidential, if it's public fix should be placed on GitLab canonical and no backports are required. -- [ ] Mark this [issue as related] to the Security Release Tracking Issue. You can find it on the topic of the `#releases` Slack channel. +- [ ] Mark this [issue as linked] to the Security Release Tracking Issue. You can find it on the topic of the `#releases` Slack channel. - Fill out the [Links section](#links): - [ ] Next to **Issue on GitLab**, add a link to the `gitlab-org/gitlab` issue that describes the security vulnerability. +- [ ] Add one of the `~severity::x` labels to the issue and all associated merge requests. ## Development @@ -64,6 +65,6 @@ After your merge request has been approved according to our [approval guidelines [secpick documentation]: https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/security/utilities/secpick_script.md [security Release merge request template]: https://gitlab.com/gitlab-org/security/gitlab/blob/master/.gitlab/merge_request_templates/Security%20Release.md [approval guidelines]: https://docs.gitlab.com/ee/development/code_review.html#approval-guidelines -[issue as related]: https://docs.gitlab.com/ee/user/project/issues/related_issues.html#adding-a-related-issue +[issue as linked]: https://docs.gitlab.com/ee/user/project/issues/related_issues.html#add-a-linked-issue /label ~security diff --git a/.gitlab/issue_templates/experiment_tracking_template.md b/.gitlab/issue_templates/experiment_tracking_template.md index 432ae57e594..c653a3a2d40 100644 --- a/.gitlab/issue_templates/experiment_tracking_template.md +++ b/.gitlab/issue_templates/experiment_tracking_template.md @@ -27,7 +27,7 @@ As well as defining the experiment rollout and cleanup, this issue incorporates ### What might happen if this goes wrong? ### What can we monitor to detect problems with this? -<!-- Which dashboards from https://dashboards.gitlab.net are most relevant? Sentry errors reports can alse be useful to review --> +<!-- Which dashboards from https://dashboards.gitlab.net are most relevant? Sentry errors reports can also be useful to review --> ### Tracked data <!-- brief description or link to issue or Sisense dashboard --> @@ -81,6 +81,7 @@ If applicable, any groups/projects that are happy to have this feature turned on - [ ] Announce on the issue that the flag has been enabled - [ ] Remove experiment code and feature flag and add changelog entry - a separate [cleanup issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Experiment%20Successful%20Cleanup) might be required - [ ] After the flag removal is deployed, [clean up the feature flag](https://docs.gitlab.com/ee/development/feature_flags/controls.html#cleaning-up) by running chatops command in `#production` channel +- [ ] Assign to the product manager to update the [knowledge base](https://about.gitlab.com/direction/growth/#growth-insights-knowledge-base) (if applicable) ## Rollback Steps diff --git a/.gitlab/merge_request_templates/New End To End Test.md b/.gitlab/merge_request_templates/New End To End Test.md index c38c3ce340a..9e6c4049b90 100644 --- a/.gitlab/merge_request_templates/New End To End Test.md +++ b/.gitlab/merge_request_templates/New End To End Test.md @@ -10,7 +10,7 @@ Please link to the respective test case in the testcases project - [ ] Note if the test is intended to run in specific scenarios. If a scenario is new, add a link to the MR that adds the new scenario. - [ ] Follow the end-to-end tests [style guide](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/style_guide.html) and [best practices](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/best_practices.html). - [ ] Use the appropriate [RSpec metadata tag(s)](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/rspec_metadata_tests.html#rspec-metadata-for-end-to-end-tests). -- [ ] Ensure that a created resource is removed after test execution. +- [ ] Ensure that a created resource is removed after test execution. A `Group` resource can be shared between multiple tests. Do not remove it unless it has a unique path. Note that we have a cleanup job that periodically removes groups under `gitlab-qa-sandbox-group`. - [ ] Ensure that no [transient bugs](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#transient-bugs) are hidden accidentally due to the usage of `waits` and `reloads`. - [ ] Verify the tags to ensure it runs on the desired test environments. - [ ] If this MR has a dependency on another MR, such as a GitLab QA MR, specify the order in which the MRs should be merged. diff --git a/.gitlab/merge_request_templates/Quarantine End to End Test.md b/.gitlab/merge_request_templates/Quarantine End to End Test.md new file mode 100644 index 00000000000..5794a62df96 --- /dev/null +++ b/.gitlab/merge_request_templates/Quarantine End to End Test.md @@ -0,0 +1,44 @@ +## What does this MR do? + +<!-- +Please describe why the end-to-end test is being quarantined/ de-quarantined. + +Please note that the aim of quarantining a test is not to get back a green pipeline, but rather to reduce +the noise (due to constantly failing tests, flaky tests, and so on) so that new failures are not missed. +--> + + +### E2E Test Failure issue(s) + +<!-- Please link to the respective E2E test failure issue. --> + + +### Check-list + +- [ ] General code guidelines check-list + - [ ] [Code review guidelines](https://docs.gitlab.com/ee/development/code_review.html) + - [ ] [Style guides](https://docs.gitlab.com/ee/development/contributing/style_guides.html) +- [ ] Quarantine test check-list + - [ ] Follow the [Quarantining Tests guide](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests). + - [ ] Confirm the test has a [`quarantine:` tag with the specified quarantine type](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantined-test-types). + - [ ] Note if the test should be [quarantined for a specific environment](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/environment_selection.html#quarantining-a-test-for-a-specific-environment). +- [ ] Dequarantine test check-list + - [ ] Follow the [Dequarantining Tests guide](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#dequarantining-tests). + - [ ] Confirm the test consistently passes on the target GitLab environment(s). + - [ ] (Optionally) [Trigger a manual GitLab-QA pipeline](https://about.gitlab.com/handbook/engineering/quality/guidelines/tips-and-tricks/#running-gitlab-qa-pipeline-against-a-specific-gitlab-release) against a specific GitLab environment using the `RELEASE` variable from the `package-and-qa` job of the current merge request. +- [ ] To ensure a faster turnaround, ask in the `#quality` Slack channel for someone to review and merge the merge request, rather than assigning it directly. + +<!-- Base labels. --> +/label ~"Quality" ~"QA" ~"feature" ~"feature::maintenance" + +<!-- Labels to pick into auto-deploy. --> +/label ~"Pick into auto-deploy" ~"priority::1" ~"severity::1" + +<!-- +Choose the stage that appears in the test path, e.g. ~"devops::create" for +`qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb`. +--> +/label ~devops:: + +<!-- Select the current milestone. --> +/milestone % |