Merge branch 'master' into update-todo-in-uiupdate-todo-in-ui

* master: (435 commits)
  Change occurrence of Sidekiq::Testing.inline!
  Fix order-dependent spec failure in appearance_spec.rb
  Put a failed example from appearance_spec in quarantine
  Cache PerformanceBar.allowed_user_ids list locally and in Redis
  Add Grafana to Admin > Monitoring menu when enabled
  Add changelog entry
  Add salesforce logo
  Move error_tracking_frontend specs to Jest
  Only save Peek session in Redis when Peek is enabled
  Migrate markdown header_spec.js to Jest
  Fix golint command in Go guide doc to be recursive
  Move images to their own dirs
  Gitlab -> GitLab
  Re-align CE and EE API docs
  Rename Release groups in issue_workflow.md
  Update api docs to finish aligning EE and CE docs
  Update locale.pot
  Update TODO: allow_collaboration column renaming
  Show upcoming status for releases
  Rebased and squashed commits
  ...

8 files changed, 129 insertions, 98 deletions

diff --git a/scripts/generate-gems-memory-metrics-static b/scripts/generate-gems-memory-metrics-static
new file mode 100755
index 00000000000..aa7ce3615bf
--- /dev/null
+++ b/scripts/generate-gems-memory-metrics-static
@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+abort "usage: #{__FILE__} <memory_bundle_objects_file_name>" unless ARGV.length == 1
+memory_bundle_objects_file_name = ARGV.first
+
+full_report = File.readlines(memory_bundle_objects_file_name)
+
+allocated_str = full_report[1]
+retained_str = full_report[2]
+allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str)
+retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str)
+
+abort 'failed to process the benchmark output' unless allocated_stats && retained_stats
+
+puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
+puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
+puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}"
+puts "memory_static_objects_retained_items #{retained_stats[:objects]}"
diff --git a/scripts/generate-gems-size-metrics-static b/scripts/generate-gems-size-metrics-static
new file mode 100755
index 00000000000..ceec8aaccf1
--- /dev/null
+++ b/scripts/generate-gems-size-metrics-static
@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1
+memory_bundle_mem_file_name = ARGV.first
+
+full_report = File.readlines(memory_bundle_mem_file_name)
+
+def total_size(memory_bundle_mem_report)
+  stats = /TOP: (?<total_mibs_str>.*) MiB/.match(memory_bundle_mem_report.first)
+  abort 'failed to process the benchmark output' unless stats
+  "gem_total_size_mb #{stats[:total_mibs_str].to_f.round(1)}"
+end
+
+TOP_LEVEL_GEM_LOG_FORMAT = /^  (?<gem_name>\S.*):\s*(?<gem_size>\d[.\d]*)\s*MiB/.freeze
+def all_gems(memory_bundle_mem_report)
+  memory_bundle_mem_report.map do |line|
+    TOP_LEVEL_GEM_LOG_FORMAT.match(line)
+  end.compact
+end
+
+def gems_as_metrics(gems_match_data)
+  gems_match_data.map do |gem|
+    gem_name = gem[:gem_name]
+    gem_size_mb = gem[:gem_size].to_f.round(1)
+    "gem_size_mb{name=\"#{gem_name}\"} #{gem_size_mb}"
+  end
+end
+
+puts total_size(full_report)
+puts gems_as_metrics(all_gems(full_report)).sort(&:casecmp)
diff --git a/scripts/generate-memory-metrics-on-boot b/scripts/generate-memory-metrics-on-boot
new file mode 100755
index 00000000000..5197a8fcdcd
--- /dev/null
+++ b/scripts/generate-memory-metrics-on-boot
@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1
+memory_bundle_mem_file_name = ARGV.first
+
+full_report = File.open(memory_bundle_mem_file_name).read
+
+stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report)
+abort 'failed to process the benchmark output' unless stats
+
+puts "total_memory_used_by_dependencies_on_boot_prod_env_mb #{stats[:total_mibs_str].to_f.round(1)}"
diff --git a/scripts/memory-static b/scripts/memory-static
deleted file mode 100755
index 54f147a7a91..00000000000
--- a/scripts/memory-static
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative '../lib/gitlab/popen'
-
-full_report_filename, metrics_filename = ARGV
-abort 'usage: memory-static <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename
-
-full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:mem))
-abort 'failed to execute the benchmark' unless status.zero?
-
-File.open(full_report_filename, 'w') do |f|
-  f.write(full_report)
-end
-
-stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report.lines.first)
-abort 'failed to process the benchmark output' unless stats
-
-File.open(metrics_filename, 'a') do |f|
-  f.puts "memory_static_total_mb #{stats[:total_mibs_str].to_f.round(1)}"
-end
diff --git a/scripts/memory-static-objects b/scripts/memory-static-objects
deleted file mode 100755
index 2ad38d9717c..00000000000
--- a/scripts/memory-static-objects
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative '../lib/gitlab/popen'
-
-full_report_filename, metrics_filename = ARGV
-abort 'usage: memory-static-objects <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename
-
-full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:objects))
-abort 'failed to execute the benchmark' unless status.zero?
-
-File.open(full_report_filename, 'w') do |f|
-  f.write(full_report)
-end
-
-allocated_str = full_report.lines[1]
-retained_str = full_report.lines[2]
-allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str)
-retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str)
-
-abort 'failed to process the benchmark output' unless allocated_stats && retained_stats
-
-File.open(metrics_filename, 'a') do |f|
-  f.puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
-  f.puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (104 * 1024)).round(1)}"
-  f.puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}"
-  f.puts "memory_static_objects_retained_items #{retained_stats[:objects]}"
-end
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index 9b0d5d4f719..0950ec272a5 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -35,9 +35,11 @@ sed -i 's/username: root/username: gitlab/g' config/database.yml
 
 if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
   sed -i 's/localhost/postgres/g' config/database.yml
+  sed -i 's/username: git/username: postgres/g' config/database.yml
 
   if [ -f config/database_geo.yml ]; then
     sed -i 's/localhost/postgres/g' config/database_geo.yml
+    sed -i 's/username: git/username: postgres/g' config/database_geo.yml
   fi
 else # Assume it's mysql
   sed -i 's/localhost/mysql/g' config/database.yml
@@ -48,16 +50,16 @@ else # Assume it's mysql
 fi
 
 cp config/resque.yml.example config/resque.yml
-sed -i 's/localhost/redis/g' config/resque.yml
+sed -i 's|url:.*$|url: redis://redis:6379|g' config/resque.yml
 
 cp config/redis.cache.yml.example config/redis.cache.yml
-sed -i 's/localhost/redis/g' config/redis.cache.yml
+sed -i 's|url:.*$|url: redis://redis:6379/10|g' config/redis.cache.yml
 
 cp config/redis.queues.yml.example config/redis.queues.yml
-sed -i 's/localhost/redis/g' config/redis.queues.yml
+sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml
 
 cp config/redis.shared_state.yml.example config/redis.shared_state.yml
-sed -i 's/localhost/redis/g' config/redis.shared_state.yml
+sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
 
 if [ "$SETUP_DB" != "false" ]; then
   setup_db
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 3bae2e08a6f..633ea28e96c 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -1,7 +1,7 @@
 [[ "$TRACE" ]] && set -x
 export TILLER_NAMESPACE="$KUBE_NAMESPACE"
 
-function deployExists() {
+function deploy_exists() {
   local namespace="${1}"
   local deploy="${2}"
   echoinfo "Checking if ${deploy} exists in the ${namespace} namespace..." true
@@ -13,8 +13,7 @@ function deployExists() {
   return $deploy_exists
 }
 
-function previousDeployFailed() {
-  set +e
+function previous_deploy_failed() {
   local deploy="${1}"
   echoinfo "Checking for previous deployment of ${deploy}" true
 
@@ -34,7 +33,6 @@ function previousDeployFailed() {
   else
     echoerr "Previous deployment NOT found."
   fi
-  set -e
   return $status
 }
 
@@ -51,49 +49,35 @@ function delete() {
   helm delete --purge "$name"
 }
 
-function cleanup() {
-  if [ -z "$CI_ENVIRONMENT_SLUG" ]; then
-    echoerr "No release given, aborting the delete!"
-    return
-  fi
-
-  echoinfo "Cleaning up '$CI_ENVIRONMENT_SLUG'..." true
-
-  kubectl -n "$KUBE_NAMESPACE" delete \
-    ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa \
-    --now --ignore-not-found --include-uninitialized \
-    -l release="$CI_ENVIRONMENT_SLUG"
-}
-
 function get_pod() {
   local app_name="${1}"
   local status="${2-Running}"
   get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name"
-  echoinfo "Running '${get_pod_cmd}'" true
+  echoinfo "Waiting till '${app_name}' pod is ready" true
+  echoinfo "Running '${get_pod_cmd}'"
 
+  local interval=5
+  local elapsed_seconds=0
+  local max_seconds=$((2 * 60))
   while true; do
     local pod_name
     pod_name="$(eval "${get_pod_cmd}")"
     [[ "${pod_name}" == "" ]] || break
 
-    echoinfo "Waiting till '${app_name}' pod is ready";
-    sleep 5;
+    if [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
+      echoerr "The pod name couldn't be found after ${elapsed_seconds} seconds, aborting."
+      break
+    fi
+
+    printf "."
+    let "elapsed_seconds+=interval"
+    sleep ${interval}
   done
 
   echoinfo "The pod name is '${pod_name}'."
   echo "${pod_name}"
 }
 
-function perform_review_app_deployment() {
-  check_kube_domain
-  ensure_namespace
-  install_tiller
-  install_external_dns
-  time deploy
-  wait_for_review_app_to_be_accessible
-  add_license
-}
-
 function check_kube_domain() {
   echoinfo "Checking that Kube domain exists..." true
 
@@ -119,9 +103,16 @@ function install_tiller() {
   echoinfo "Initiating the Helm client..."
   helm init --client-only
 
+  # Set toleration for Tiller to be installed on a specific node pool
   helm init \
+    --wait \
     --upgrade \
-    --replicas 2
+    --node-selectors "app=helm" \
+    --replicas 3 \
+    --override "spec.template.spec.tolerations[0].key"="dedicated" \
+    --override "spec.template.spec.tolerations[0].operator"="Equal" \
+    --override "spec.template.spec.tolerations[0].value"="helm" \
+    --override "spec.template.spec.tolerations[0].effect"="NoSchedule"
 
   kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
 
@@ -137,7 +128,7 @@ function install_external_dns() {
   domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}')
   echoinfo "Installing external DNS for domain ${domain}..." true
 
-  if ! deployExists "${KUBE_NAMESPACE}" "${release_name}" || previousDeployFailed "${release_name}" ; then
+  if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${release_name}" ; then
     echoinfo "Installing external-dns Helm chart"
     helm repo update
     helm install stable/external-dns \
@@ -156,7 +147,7 @@ function install_external_dns() {
   fi
 }
 
-function create_secret() {
+function create_application_secret() {
   echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password secret in the ${KUBE_NAMESPACE} namespace..." true
 
   kubectl create secret generic -n "$KUBE_NAMESPACE" \
@@ -165,7 +156,7 @@ function create_secret() {
     --dry-run -o json | kubectl apply -f -
 }
 
-function download_gitlab_chart() {
+function download_chart() {
   echoinfo "Downloading the GitLab chart..." true
 
   curl -o gitlab.tar.bz2 "https://gitlab.com/charts/gitlab/-/archive/${GITLAB_HELM_CHART_REF}/gitlab-${GITLAB_HELM_CHART_REF}.tar.bz2"
@@ -194,14 +185,12 @@ function deploy() {
   gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}"
 
   # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
-  if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previousDeployFailed "$CI_ENVIRONMENT_SLUG" ; then
+  if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then
     echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
     delete
-    cleanup
   fi
 
-  create_secret
-  download_gitlab_chart
+  create_application_secret
 
 HELM_CMD=$(cat << EOF
   helm upgrade --install \
@@ -216,7 +205,7 @@ HELM_CMD=$(cat << EOF
     --set prometheus.install=false \
     --set global.ingress.configureCertmanager=false \
     --set global.ingress.tls.secretName=tls-cert \
-    --set global.ingress.annotations."external-dns\.alpha\.kubernetes\.io/ttl"="10"
+    --set global.ingress.annotations."external-dns\.alpha\.kubernetes\.io/ttl"="10" \
     --set nginx-ingress.controller.service.enableHttp=false \
     --set nginx-ingress.defaultBackend.resources.requests.memory=7Mi \
     --set nginx-ingress.controller.resources.requests.memory=440M \
@@ -252,14 +241,35 @@ EOF
   echoinfo "Deploying with:"
   echoinfo "${HELM_CMD}"
 
-  eval $HELM_CMD || true
+  eval "${HELM_CMD}"
+}
+
+function display_deployment_debug() {
+  migrations_pod=$(get_pod "migrations");
+  if [ -z "${migrations_pod}" ]; then
+    echoerr "Migrations pod not found."
+  else
+    echoinfo "Logs tail of the ${migrations_pod} pod..."
+
+    kubectl logs -n "$KUBE_NAMESPACE" "${migrations_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
+  fi
+
+  unicorn_pod=$(get_pod "unicorn");
+  if [ -z "${unicorn_pod}" ]; then
+    echoerr "Unicorn pod not found."
+  else
+    echoinfo "Logs tail of the ${unicorn_pod} pod..."
+
+    kubectl logs -n "$KUBE_NAMESPACE" -c unicorn "${unicorn_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
+  fi
 }
 
 function wait_for_review_app_to_be_accessible() {
-  # In case the Review App isn't completely available yet. Keep trying for 5 minutes.
+  echoinfo "Waiting for the Review App at ${CI_ENVIRONMENT_URL} to be accessible..." true
+
   local interval=5
   local elapsed_seconds=0
-  local max_seconds=$((5 * 60))
+  local max_seconds=$((2 * 60))
   while true; do
     local review_app_http_code
     review_app_http_code=$(curl --silent --output /dev/null --max-time 5 --write-out "%{http_code}" "${CI_ENVIRONMENT_URL}/users/sign_in")
@@ -272,10 +282,10 @@ function wait_for_review_app_to_be_accessible() {
     sleep ${interval}
   done
 
-  if [[ "${review_app_http_code}" == "200" ]]; then
-    echoinfo "The Review App at ${CI_ENVIRONMENT_URL} is ready!"
+  if [[ "${review_app_http_code}" -eq "200" ]]; then
+    echoinfo "The Review App at ${CI_ENVIRONMENT_URL} is ready after ${elapsed_seconds} seconds!"
   else
-    echoerr "The Review App at ${CI_ENVIRONMENT_URL} isn't ready after 5 minutes of polling..."
+    echoerr "The Review App at ${CI_ENVIRONMENT_URL} isn't ready after ${max_seconds} seconds of polling..."
     exit 1
   fi
 }
diff --git a/scripts/trigger-build b/scripts/trigger-build
index 52bc61cac56..4d8110fce10 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -122,7 +122,14 @@ module Trigger
     end
 
     def ref
-      ENV['CNG_BRANCH'] || 'master'
+      default_ref =
+        if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/
+          ENV['CI_COMMIT_REF_NAME']
+        else
+          'master'
+        end
+
+      ENV['CNG_BRANCH'] || default_ref
     end
 
     def trigger_token