Add latest changes from gitlab-org/gitlab@13-2-stable-ee

12 files changed, 115 insertions, 33 deletions

diff --git a/scripts/frontend/merge_coverage_frontend.js b/scripts/frontend/merge_coverage_frontend.js
index 507695b45e5..99034176b29 100644
--- a/scripts/frontend/merge_coverage_frontend.js
+++ b/scripts/frontend/merge_coverage_frontend.js
@@ -26,6 +26,6 @@ reportFiles
 
 const context = createContext({ coverageMap: coverageMap, dir: 'coverage-frontend' });
 
-['json', 'lcov', 'text-summary', 'clover'].forEach(reporter => {
+['json', 'lcov', 'text-summary', 'clover', 'cobertura'].forEach(reporter => {
   create(reporter, {}).execute(context);
 });
diff --git a/scripts/frontend/prettier.js b/scripts/frontend/prettier.js
index 45cdef2ba86..7772f80c233 100644
--- a/scripts/frontend/prettier.js
+++ b/scripts/frontend/prettier.js
@@ -3,7 +3,7 @@ const prettier = require('prettier');
 const fs = require('fs');
 const { getStagedFiles } = require('./frontend_script_utils');
 
-const matchExtensions = ['js', 'vue'];
+const matchExtensions = ['js', 'vue', 'graphql'];
 
 // This will improve glob performance by excluding certain directories.
 // The .prettierignore file will also be respected, but after the glob has executed.
diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb
index c69c4ea747b..54bf07b3773 100644
--- a/scripts/gitaly_test.rb
+++ b/scripts/gitaly_test.rb
@@ -6,8 +6,16 @@
 
 require 'securerandom'
 require 'socket'
+require 'logger'
 
 module GitalyTest
+  LOGGER = begin
+             default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
+             level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
+             level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse
+             Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg })
+           end
+
   def tmp_tests_gitaly_dir
     File.expand_path('../tmp/tests/gitaly', __dir__)
   end
@@ -98,7 +106,7 @@ module GitalyTest
   end
 
   def check_gitaly_config!
-    puts "Checking gitaly-ruby Gemfile..."
+    LOGGER.debug "Checking gitaly-ruby Gemfile...\n"
 
     unless File.exist?(gemfile)
       message = "#{gemfile} does not exist."
@@ -106,8 +114,9 @@ module GitalyTest
       abort message
     end
 
-    puts 'Checking gitaly-ruby bundle...'
-    abort 'bundle check failed' unless system(env, 'bundle', 'check', chdir: File.dirname(gemfile))
+    LOGGER.debug "Checking gitaly-ruby bundle...\n"
+    out = ENV['CI'] ? STDOUT : '/dev/null'
+    abort 'bundle check failed' unless system(env, 'bundle', 'check', out: out, chdir: File.dirname(gemfile))
   end
 
   def read_socket_path(service)
@@ -126,22 +135,22 @@ module GitalyTest
   end
 
   def try_connect!(service)
-    print "Trying to connect to #{service}: "
+    LOGGER.debug "Trying to connect to #{service}: "
     timeout = 20
     delay = 0.1
     socket = read_socket_path(service)
 
     Integer(timeout / delay).times do
       UNIXSocket.new(socket)
-      puts ' OK'
+      LOGGER.debug " OK\n"
 
       return
     rescue Errno::ENOENT, Errno::ECONNREFUSED
-      print '.'
+      LOGGER.debug '.'
       sleep delay
     end
 
-    puts ' FAILED'
+    LOGGER.warn " FAILED to connect to #{service}\n"
 
     raise "could not connect to #{socket}"
   end
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index ca9747987df..72e6334d0fc 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -41,14 +41,15 @@ fi
 
 # Do not use 'README.md', instead use 'index.md'
 # Number of 'README.md's as of 2020-05-28
-NUMBER_READMES=45
+NUMBER_READMES=44
 FIND_READMES=$(find doc/ -name "README.md" | wc -l)
 echo '=> Checking for new README.md files...'
 echo
 if [ ${FIND_READMES} -ne $NUMBER_READMES ]
 then
   echo
-  echo '  ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2
+  echo '  ✖ ERROR: The number of README.md file(s) has changed. Use index.md instead of README.md.' >&2
+  echo '  ✖        If removing a README.md file, update NUMBER_READMES in lint-doc.sh.' >&2
   echo '  https://docs.gitlab.com/ee/development/documentation/styleguide.html#work-with-directories-and-files'
   echo
   ((ERRORCODE++))
diff --git a/scripts/merge-simplecov b/scripts/merge-simplecov
index 746be3317a7..c00dae81c4d 100755
--- a/scripts/merge-simplecov
+++ b/scripts/merge-simplecov
@@ -2,6 +2,7 @@
 
 require_relative '../spec/simplecov_env'
 SimpleCovEnv.configure_profile
+SimpleCovEnv.configure_formatter
 
 module SimpleCov
   module ResultMerger
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index d2cc16f4f8b..1243609dc24 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -2,7 +2,7 @@
 
 export SETUP_DB=${SETUP_DB:-true}
 export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
-export BUNDLE_INSTALL_FLAGS="--without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet"
+export BUNDLE_INSTALL_FLAGS=${BUNDLE_INSTALL_FLAGS:-"--without=production --without=development --jobs=$(nproc) --path=vendor --retry=3 --quiet"}
 
 if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
   bundle --version
@@ -11,7 +11,7 @@ if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
   # When we test multiple versions of PG in the same pipeline, we have a single `setup-test-env`
   # job but the `pg` gem needs to be rebuilt since it includes extensions (https://guides.rubygems.org/gems-with-extensions).
   # Uncomment the following line if multiple versions of PG are tested in the same pipeline.
-  # run_timed_command "bundle pristine pg"
+  run_timed_command "bundle pristine pg"
 fi
 
 # Only install knapsack after bundle install! Otherwise oddly some native
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index a9659071a2f..e40c6cd7276 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require 'gitlab'
-require_relative File.expand_path('../../lib/quality/helm3_client.rb', __dir__)
-require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__)
+require_relative File.expand_path('../../tooling/lib/tooling/helm3_client.rb', __dir__)
+require_relative File.expand_path('../../tooling/lib/tooling/kubernetes_client.rb', __dir__)
 
 class AutomatedCleanup
   attr_reader :project_path, :gitlab_token
@@ -40,15 +40,15 @@ class AutomatedCleanup
   end
 
   def review_apps_namespace
-    self.class.ee? ? 'review-apps-ee' : 'review-apps-ce'
+    'review-apps'
   end
 
   def helm
-    @helm ||= Quality::Helm3Client.new(namespace: review_apps_namespace)
+    @helm ||= Tooling::Helm3Client.new(namespace: review_apps_namespace)
   end
 
   def kubernetes
-    @kubernetes ||= Quality::KubernetesClient.new(namespace: review_apps_namespace)
+    @kubernetes ||= Tooling::KubernetesClient.new(namespace: review_apps_namespace)
   end
 
   def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:)
@@ -76,7 +76,7 @@ class AutomatedCleanup
       if deployed_at < delete_threshold
         deleted_environment = delete_environment(environment, deployment)
         if deleted_environment
-          release = Quality::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace)
+          release = Tooling::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace)
           releases_to_delete << release
         end
       else
@@ -157,11 +157,11 @@ class AutomatedCleanup
     helm.delete(release_name: releases_names)
     kubernetes.cleanup(release_name: releases_names, wait: false)
 
-  rescue Quality::Helm3Client::CommandFailedError => ex
+  rescue Tooling::Helm3Client::CommandFailedError => ex
     raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS)
 
     puts "Ignoring the following Helm error:\n#{ex}\n"
-  rescue Quality::KubernetesClient::CommandFailedError => ex
+  rescue Tooling::KubernetesClient::CommandFailedError => ex
     raise ex unless ignore_exception?(ex.message, IGNORED_KUBERNETES_ERRORS)
 
     puts "Ignoring the following Kubernetes error:\n#{ex}\n"
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 6fb6943fb90..82be2d3a691 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -7,7 +7,7 @@ global:
       external-dns.alpha.kubernetes.io/ttl: 10
     configureCertmanager: false
     tls:
-      secretName: tls-cert
+      secretName: review-apps-tls
   initialRootPassword:
     secret: shared-gitlab-initial-root-password
 certmanager:
@@ -61,11 +61,11 @@ gitlab:
   task-runner:
     resources:
       requests:
-        cpu: 50m
-        memory: 350M
+        cpu: 300m
+        memory: 800M
       limits:
-        cpu: 100m
-        memory: 700M
+        cpu: 450m
+        memory: 1200M
   webservice:
     resources:
       requests:
@@ -137,10 +137,10 @@ postgresql:
     enabled: false
   resources:
     requests:
-      cpu: 347m
+      cpu: 550m
       memory: 250M
     limits:
-      cpu: 520m
+      cpu: 825m
       memory: 375M
 prometheus:
   install: false
diff --git a/scripts/review_apps/gcp_cleanup.sh b/scripts/review_apps/gcp_cleanup.sh
index f289a50f629..3225631e8c7 100755
--- a/scripts/review_apps/gcp_cleanup.sh
+++ b/scripts/review_apps/gcp_cleanup.sh
@@ -11,7 +11,7 @@ function setup_gcp_dependencies() {
 
 # These scripts require the following environment variables:
 # - REVIEW_APPS_GCP_REGION - e.g `us-central1`
-# - KUBE_NAMESPACE - e.g `review-apps-ee`
+# - KUBE_NAMESPACE - e.g `review-apps`
 
 function delete_firewall_rules() {
   if [[ ${#@} -eq 0 ]]; then
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 1214ee5f462..74291f6aef4 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -66,7 +66,7 @@ function kubectl_cleanup_release() {
   local release="${2}"
 
   echoinfo "Deleting all K8s resources matching '${release}'..." true
-  kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
+  kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
     | grep "${release}" \
     | awk '{print $1}' \
     | xargs kubectl --namespace "${namespace}" delete \
@@ -126,6 +126,38 @@ function get_pod() {
   echo "${pod_name}"
 }
 
+function run_task() {
+  local namespace="${KUBE_NAMESPACE}"
+  local ruby_cmd="${1}"
+  local task_runner_pod=$(get_pod "task-runner")
+
+  kubectl exec -it --namespace "${namespace}" "${task_runner_pod}" -- gitlab-rails runner "${ruby_cmd}"
+}
+
+function disable_sign_ups() {
+  if [ -z ${REVIEW_APPS_ROOT_TOKEN+x} ]; then
+    echoerr "In order to protect Review Apps, REVIEW_APPS_ROOT_TOKEN variable must be set"
+    false
+  else
+    true
+  fi
+
+  # Create the root token
+  local ruby_cmd="token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api], name: 'Token to disable sign-ups'); token.set_token('${REVIEW_APPS_ROOT_TOKEN}'); begin; token.save!; rescue(ActiveRecord::RecordNotUnique); end"
+  run_task "${ruby_cmd}"
+
+  # Disable sign-ups
+  curl  --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false"
+
+  local signup_enabled=$(curl --silent --show-error --request GET --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings" | jq ".signup_enabled")
+  if [[ "${signup_enabled}" == "false" ]]; then
+    echoinfo "Sign-ups have been disabled successfully."
+  else
+    echoerr "Sign-ups should be disabled but are still enabled!"
+    false
+  fi
+}
+
 function check_kube_domain() {
   echoinfo "Checking that Kube domain exists..." true
 
@@ -181,6 +213,32 @@ function install_external_dns() {
   fi
 }
 
+# This script is used to install cert-manager in the cluster
+# The installation steps are documented in
+# https://gitlab.com/gitlab-org/quality/team-tasks/snippets/1990286
+function install_certmanager() {
+  local namespace="${KUBE_NAMESPACE}"
+  local release="cert-manager-review-app-helm3"
+
+  echoinfo "Installing cert-manager..." true
+
+  if ! deploy_exists "${namespace}" "${release}" || previous_deploy_failed "${namespace}" "${release}" ; then
+    kubectl apply \
+    -f https://raw.githubusercontent.com/jetstack/cert-manager/release-0.10/deploy/manifests/00-crds.yaml
+
+    echoinfo "Installing cert-manager Helm chart"
+    helm repo add jetstack https://charts.jetstack.io
+    helm repo update
+
+    helm install "${release}" jetstack/cert-manager \
+      --namespace "${namespace}" \
+      --version v0.15.1 \
+      --set installCRDS=true
+  else
+    echoinfo "The cert-manager Helm chart is already successfully deployed."
+  fi
+}
+
 function create_application_secret() {
   local namespace="${KUBE_NAMESPACE}"
   local release="${CI_ENVIRONMENT_SLUG}"
@@ -233,6 +291,17 @@ function base_config_changed() {
   curl "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/changes" | jq '.changes | any(.old_path == "scripts/review_apps/base-config.yaml")'
 }
 
+function parse_gitaly_image_tag() {
+  local gitaly_version="${GITALY_VERSION}"
+
+  # prepend semver version with `v`
+  if [[ $gitaly_version =~  ^[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?(-ee)?$ ]]; then
+    echo "v${gitaly_version}"
+  else
+    echo "${gitaly_version}"
+  fi
+}
+
 function deploy() {
   local namespace="${KUBE_NAMESPACE}"
   local release="${CI_ENVIRONMENT_SLUG}"
@@ -248,6 +317,7 @@ function deploy() {
   gitlab_webservice_image_repository="${IMAGE_REPOSITORY}/gitlab-webservice-ee"
   gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-ee"
   gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
+  gitaly_image_tag=$(parse_gitaly_image_tag)
   gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
   gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-ee"
 
@@ -269,7 +339,7 @@ HELM_CMD=$(cat << EOF
     --set gitlab.migrations.image.repository="${gitlab_migrations_image_repository}" \
     --set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \
     --set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \
-    --set gitlab.gitaly.image.tag="v${GITALY_VERSION}" \
+    --set gitlab.gitaly.image.tag="${gitaly_image_tag}" \
     --set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \
     --set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \
     --set gitlab.sidekiq.annotations.commit="${CI_COMMIT_SHORT_SHA}" \
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 110567a15be..1ecf9a566d7 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -119,12 +119,12 @@ function rspec_matched_tests() {
   local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'})
 
   if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then
-    echo "There are more than ${test_file_count_threshold} FOSS test files matched,"
+    echo "This job is intentionally failed because there are more than ${test_file_count_threshold} FOSS test files matched,"
     echo "which would take too long to run in this job."
     echo "To reduce the likelihood of breaking FOSS pipelines,"
     echo "please add [RUN AS-IF-FOSS] to the MR title and restart the pipeline."
     echo "This would run all as-if-foss jobs in this merge request"
-    echo "and remove this job from the pipeline."
+    echo "and remove this failing job from the pipeline."
     exit 1
   fi
 
diff --git a/scripts/trigger-build b/scripts/trigger-build
index b8bea95a069..9f0df21e7f1 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -184,6 +184,7 @@ module Trigger
       edition = Trigger.ee? ? 'EE' : 'CE'
 
       {
+        "ee" => Trigger.ee? ? "true" : "false",
         "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'],
         "GITLAB_TAG" => ENV['CI_COMMIT_TAG'],
         "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'],