Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

10 files changed, 106 insertions, 99 deletions

diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml
index ae36c0cea70..c439e9a7c80 100644
--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
@@ -44,7 +44,7 @@ docs-lint markdown:
     - .default-retry
     - .docs:rules:docs-lint
   # When updating the image version here, update it in /scripts/lint-doc.sh too.
-  image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.12.0-markdownlint-0.29.0
+  image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.15-vale-2.14.0-markdownlint-0.30.0
   stage: lint
   needs: []
   script:
@@ -53,7 +53,7 @@ docs-lint markdown:
 docs-lint links:
   extends:
     - .docs:rules:docs-lint
-  image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.5-08847baa
+  image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.15-ruby-2.7.5-cee62c13
   stage: lint
   needs: []
   script:
@@ -77,15 +77,16 @@ ui-docs-links lint:
   script:
     - bundle exec haml-lint -i DocumentationLinks
 
-docs-lint deprecations:
+docs-lint deprecations-and-removals:
   variables:
     SETUP_DB: "false"
   extends:
     - .default-retry
     - .rails-cache
     - .default-before_script
-    - .docs:rules:deprecations
+    - .docs:rules:deprecations-and-removals
   stage: lint
   needs: []
   script:
     - bundle exec rake gitlab:docs:check_deprecations
+    - bundle exec rake gitlab:docs:check_removals
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index ea4ae3b0492..1dd5285e0ae 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -306,6 +306,11 @@ qa-frontend-node:latest:
     - .qa-frontend-node
     - .frontend:rules:qa-frontend-node-latest
   image: ${GITLAB_DEPENDENCY_PROXY}node:latest
+  # This is a workaround for https://github.com/webpack/webpack/issues/14532 until
+  # we can upgrade to Webpack 5 and switch to SHA256: https://gitlab.com/gitlab-org/gitlab/-/issues/350120
+  script:
+    - *yarn-install
+    - run_timed_command "retry yarn run webpack-prod-node-latest"
 
 webpack-dev-server:
   extends:
diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml
index 4fec223e66d..b12f76f2823 100644
--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
@@ -33,6 +33,15 @@ qa:selectors:
   script:
     - bundle exec bin/qa Test::Sanity::Selectors
 
+qa:auto_quarantine:
+  extends:
+    - .qa-job-base
+  rules:
+    - if: '$QA_TRIGGER_AUTO_QUARANTINE =~ /true|yes|1/i'
+  script:
+    - bundle exec confiner -r .confiner/quarantine.yml
+  allow_failure: true
+
 qa:selectors-as-if-foss:
   extends:
     - qa:selectors
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index d676dc2f331..1d2f94b616d 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -346,7 +346,7 @@ rspec fast_spec_helper minimal:
 db:rollback:
   extends: .db-job-base
   script:
-    - scripts/db_tasks db:migrate VERSION=20181228175414
+    - scripts/db_tasks db:migrate VERSION=20210301200959
     - scripts/db_tasks db:migrate SKIP_SCHEMA_VERSION_CHECK=true
 
 db:rollback decomposed:
@@ -360,6 +360,12 @@ db:migrate:reset:
   script:
     - bundle exec rake db:migrate:reset
 
+db:migrate:reset decomposed:
+  extends:
+    - db:migrate:reset
+    - .decomposed-database
+    - .rails:rules:decomposed-databases
+
 db:migrate-from-previous-major-version:
   extends: .db-job-base
   variables:
@@ -457,7 +463,7 @@ db:backup_and_restore:
   script:
     - . scripts/prepare_build.sh
     - bundle exec rake db:drop db:create db:structure:load db:seed_fu
-    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry}
+    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry,packages}
     - bundle exec rake gitlab:backup:create
     - date
     - bundle exec rake gitlab:backup:restore
@@ -592,8 +598,10 @@ rspec:undercoverage:
       else
         echo "Using \$CI_COMMIT_SHA ($CI_COMMIT_SHA) for this non-merge result pipeline.";
       fi;
+    - UNDERCOVERAGE_COMPARE="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-$(git merge-base origin/master HEAD)}"
+    - echo "Undercoverage comparing with ${UNDERCOVERAGE_COMPARE}"
     - if [ -f scripts/undercoverage ]; then
-        run_timed_command "scripts/undercoverage";
+        run_timed_command "scripts/undercoverage ${UNDERCOVERAGE_COMPARE}";
       fi;
 
 rspec:feature-flags:
diff --git a/.gitlab/ci/review-apps/dast.gitlab-ci.yml b/.gitlab/ci/review-apps/dast.gitlab-ci.yml
index 512c850b7da..d0ad4d23a82 100644
--- a/.gitlab/ci/review-apps/dast.gitlab-ci.yml
+++ b/.gitlab/ci/review-apps/dast.gitlab-ci.yml
@@ -45,7 +45,6 @@
 # 10019, 10021	Missing security headers
 # 10023, 10024, 10025, 10037 Information Disclosure
 # 10040	Secure Pages Include Mixed Content
-# 10055	CSP
 # 10056	X-Debug-Token Information Leak
 # Duration: 14 minutes 20 seconds
 
@@ -54,7 +53,7 @@ dast:secureHeaders-csp-infoLeak:
     - .dast_conf
   variables:
     DAST_USERNAME: "user1"
-    DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10055,10056"
+    DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10056"
   script:
     - /analyze
 
diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml
index af4674b802b..4ef6efa2604 100644
--- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml
+++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml
@@ -26,35 +26,22 @@
     - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
     - cd qa
-    - if [ -n "$KNAPSACK_REPORT_PATH" ]; then
-        bundle exec rake knapsack:download;
-      fi
-  artifacts:
-    paths:
-      - qa/tmp
-    expire_in: 7 days
-    when: always
-
-.parallel-qa-base:
-  parallel: 5
-  variables:
-    KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb"
   script:
     - |
       bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \
         -- \
         --color --format documentation \
         --format RspecJunitFormatter --out tmp/rspec.xml
-  after_script:
-    - if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then
-        mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json;
-      fi
   artifacts:
     paths:
-      - qa/tmp # we can't merge list so need to include explicitly once more
-      - qa/knapsack/gcs/regenerated-*.json
+      - qa/tmp
     reports:
       junit: qa/tmp/rspec.xml
+    expire_in: 7 days
+    when: always
+
+.parallel-qa-base:
+  parallel: 5
 
 .allure-report-base:
   image:
@@ -79,16 +66,6 @@
         --ignore-missing-results \
         --color
 
-.knapsack-upload-base:
-  image:
-    name: ${QA_IMAGE}
-    entrypoint: [""]
-  stage: post-qa
-  before_script:
-    - cd qa
-  script:
-    - bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]'
-
 review-qa-smoke:
   extends:
     - .review-qa-base
@@ -96,8 +73,8 @@ review-qa-smoke:
   retry: 1  # This is confusing but this means "2 runs at max".
   variables:
     QA_RUN_TYPE: review-qa-smoke
-  script:
-    - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}"
+    QA_SCENARIO: Test::Instance::Smoke
+
 
 review-qa-reliable:
   extends:
@@ -108,7 +85,6 @@ review-qa-reliable:
   variables:
     QA_RUN_TYPE: review-qa-reliable
     QA_SCENARIO: Test::Instance::Reliable
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
 
 review-qa-all:
   extends:
@@ -118,7 +94,6 @@ review-qa-all:
   variables:
     QA_RUN_TYPE: review-qa-all
     QA_SCENARIO: Test::Instance::All
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
 
 review-performance:
   extends:
@@ -155,6 +130,15 @@ allure-report-qa-smoke:
     ALLURE_REPORT_PATH_PREFIX: gitlab-review-smoke
     ALLURE_JOB_NAME: review-qa-smoke
 
+allure-report-qa-reliable:
+  extends:
+    - .allure-report-base
+    - .review:rules:review-qa-reliable-report
+  needs: ["review-qa-reliable"]
+  variables:
+    ALLURE_REPORT_PATH_PREFIX: gitlab-review-reliable
+    ALLURE_JOB_NAME: review-qa-reliable
+
 allure-report-qa-all:
   extends:
     - .allure-report-base
@@ -164,18 +148,15 @@ allure-report-qa-all:
     ALLURE_REPORT_PATH_PREFIX: gitlab-review-all
     ALLURE_JOB_NAME: review-qa-all
 
-knapsack-report-qa-all:
+knapsack-report:
   extends:
-    - .knapsack-upload-base
-    - .review:rules:knapsack-report-qa-all
-  needs: ["review-qa-all"]
-  variables:
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
-
-knapsack-report-qa-reliable:
-  extends:
-    - .knapsack-upload-base
-    - .review:rules:knapsack-report-qa-reliable
-  needs: ["review-qa-reliable"]
-  variables:
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
+    - .review:rules:knapsack-report
+  image:
+    name: ${QA_IMAGE}
+    entrypoint: [""]
+  stage: post-qa
+  allow_failure: true
+  before_script:
+    - cd qa
+  script:
+    - bundle exec rake 'knapsack:upload[tmp/knapsack/*/*.json]'
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index e62de4bc6dc..008b62f6a0f 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -171,12 +171,13 @@
   - ".markdownlint.yml"
   - "scripts/lint-doc.sh"
 
-.docs-deprecations-patterns: &docs-deprecations-patterns
+.docs-deprecations-and-removals-patterns: &docs-deprecations-and-removals-patterns
   - "doc/update/deprecations.md"
-  - "data/deprecations/*.yml"
-  - "data/deprecations/templates/_deprecation_template.md.erb"
+  - "doc/update/removals.md"
+  - "data/deprecations/**/*"
+  - "data/removals/**/*"
+  - "tooling/docs/**/*"
   - "lib/tasks/gitlab/docs/compile_deprecations.rake"
-  - "tooling/deprecations/docs.rb"
 
 .bundler-patterns: &bundler-patterns
   - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
@@ -228,6 +229,9 @@
   - "vendor/assets/**/*"
   - "{,ee/,jh/}{app/assets,app/helpers,app/presenters,app/views,locale,public,symbol}/**/*"
 
+.controllers-patterns: &controllers-patterns
+  - "{,ee/,jh/}{app/controllers}/**/*"
+
 .startup-css-patterns: &startup-css-patterns
   - "{,ee/,jh/}app/assets/stylesheets/startup/**/*"
 
@@ -256,7 +260,7 @@
   - "lib/gitlab/markdown_cache/active_record/**/*"
   - "config/prometheus/common_metrics.yml"  # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer
   - "{,ee/,jh/}app/models/project_statistics.rb"  # Used to calculate sizes in migration specs
-  - "GITALY_SERVER_VERSION" # Has interactions with background migrations:https://gitlab.com/gitlab-org/gitlab/-/issues/336538
+  - "GITALY_SERVER_VERSION"  # Has interactions with background migrations:https://gitlab.com/gitlab-org/gitlab/-/issues/336538
   # CI changes
   - ".gitlab-ci.yml"
   - ".gitlab/ci/**/*"
@@ -279,7 +283,7 @@
   - ".dockerignore"
   - "qa/**/*"
 
-# Code patterns + .ci-patterns + .workhorse-patterns
+# Code patterns + .ci-patterns
 .code-patterns: &code-patterns
   - "{package.json,yarn.lock}"
   - ".browserslistrc"
@@ -541,10 +545,10 @@
       changes: *docs-patterns
       when: on_success
 
-.docs:rules:deprecations:
+.docs:rules:deprecations-and-removals:
   rules:
     - <<: *if-default-refs
-      changes: *docs-deprecations-patterns
+      changes: *docs-deprecations-and-removals-patterns
 
 ##################
 # GraphQL rules  #
@@ -1613,11 +1617,13 @@
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *frontend-patterns
     - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *controllers-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *qa-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-patterns
       when: manual
       allow_failure: true
-    - <<: *if-dot-com-gitlab-org-merge-request
-      changes: *qa-patterns
     - <<: *if-dot-com-gitlab-org-schedule
       variables:
         KNAPSACK_GENERATE_REPORT: "true"
@@ -1648,56 +1654,51 @@
   rules:
     - when: on_success
 
-# The rule needs to be duplicated between `on_success` and `on_failure`
-# because the jobs `needs` the previous job to complete.
-# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs
-# would start running before the qa jobs have started.
-# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559
+# If the needed job isn't allowed to fail, we need to use `when: always` in
+# order to keep the job always running after it.
+#
+# If the needed job is allowed to fail, we need to use both
+# `when: on_success` and `when: on_failure` in order to keep
+# the job always running after it.
+# Not that if the needed job has `when: on_success` we can use `when: always`
+# for the depending job.
+#
+# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76756
+
+# Since `review-qa-smoke` isn't allowed to fail, we need to use `when: always` for `review-qa-smoke-report`.
 .review:rules:review-qa-smoke-report:
   rules:
-    - when: on_success
-    - when: on_failure
+    - when: always
 
 .review:rules:review-qa-reliable:
   rules:
     - when: on_success
-      allow_failure: true
+
+# Since `review-qa-reliable` isn't allowed to fail, we need to use `when: always`for `review-qa-reliable-report`.
+.review:rules:review-qa-reliable-report:
+  rules:
+    - when: always
 
 .review:rules:review-qa-all:
   rules:
-    - <<: *if-merge-request-labels-run-review-app  # we explicitely don't allow the job to fail in that case
+    - <<: *if-merge-request-labels-run-review-app  # we explicitly don't allow the job to fail in that case
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-patterns
       when: manual
-      allow_failure: true # manual jobs needs to be allowd to fail, otherwise they block the pipeline
+      allow_failure: true  # manual jobs needs to be allowed to fail, otherwise they block the pipeline
     - when: on_success
       allow_failure: true
 
-# The rule needs to be duplicated between `on_success` and `on_failure`
-# because the jobs `needs` the previous job to complete.
-# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs
-# would start running before the qa jobs have started.
-# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559
+# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `review-qa-all-report`.
 .review:rules:review-qa-all-report:
   rules:
     - when: on_success
-      allow_failure: true
     - when: on_failure
-      allow_failure: true
 
-# Generate knapsack report on successful runs only
-# Reliable suite will pass most of the time so this should yield best distribution
-.review:rules:knapsack-report-qa-reliable:
-  rules:
-    - if: '$KNAPSACK_GENERATE_REPORT == "true"'
-      when: on_success
-      allow_failure: true
-
-.review:rules:knapsack-report-qa-all:
+.review:rules:knapsack-report:
   rules:
     - if: '$KNAPSACK_GENERATE_REPORT == "true"'
       when: always
-      allow_failure: true
 
 .review:rules:review-cleanup:
   rules:
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index 1eb3bd2ea41..13108ba289a 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -151,14 +151,17 @@ detect-previous-failed-tests:
 
 add-jh-folder:
   extends: .setup:rules:add-jh-folder
-  image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge
+  image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7
   stage: prepare
   before_script:
-    - apk add --no-cache --update curl bash
+    - source ./scripts/utils.sh
+    - install_gitlab_gem
   script:
-    - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/main-jh/gitlab-main-jh.tar.gz?path=jh"
+    - JH_BRANCH=$(./scripts/setup/find-jh-branch.rb)
+    - 'echo "JH_BRANCH: ${JH_BRANCH}"'
+    - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/${JH_BRANCH}/gitlab-${JH_BRANCH}.tar.gz?path=jh"
     - tar -xf "jh-folder.tar.gz"
-    - mv gitlab-main-jh-jh/jh/ ./
+    - mv "gitlab-${JH_BRANCH}-jh/jh/" ./
     - cp Gemfile.lock jh/
     - ls -l jh/
   artifacts:
diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml
index cd53adc6d4b..aab077e575b 100644
--- a/.gitlab/ci/workhorse.gitlab-ci.yml
+++ b/.gitlab/ci/workhorse.gitlab-ci.yml
@@ -4,7 +4,7 @@ workhorse:verify:
   stage: test
   needs: []
   script:
-    - make -C workhorse # test build
+    - make -C workhorse  # test build
     - make -C workhorse verify
 
 .workhorse:test:
diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml
index 590593b9d75..218dc0a7859 100644
--- a/.gitlab/ci/yaml.gitlab-ci.yml
+++ b/.gitlab/ci/yaml.gitlab-ci.yml
@@ -10,4 +10,4 @@ lint-yaml:
   variables:
     LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates
   script:
-    - yamllint -f colored $LINT_PATHS
+    - yamllint --strict -f colored $LINT_PATHS