12 files changed, 224 insertions, 80 deletions

diff --git a/scripts/create_mysql_user.sh b/scripts/create_mysql_user.sh
deleted file mode 100644
index 35f68c581f3..00000000000
--- a/scripts/create_mysql_user.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-mysql --user=root --host=mysql <<EOF
-CREATE USER IF NOT EXISTS 'gitlab'@'%';
-GRANT ALL PRIVILEGES ON gitlabhq_test.* TO 'gitlab'@'%';
-FLUSH PRIVILEGES;
-EOF
diff --git a/scripts/frontend/file_test_coverage.js b/scripts/frontend/file_test_coverage.js
new file mode 100755
index 00000000000..7d1eb45d4bc
--- /dev/null
+++ b/scripts/frontend/file_test_coverage.js
@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+
+/**
+ * Counts the number of frontend test files and compares them against the number of application files.
+ *
+ * Example output:
+ *
+ * Source files: 1551
+ * Test files: 716
+ * Coverage: 46.16%
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const sourceDirectories = ['app/assets/javascripts'];
+const testDirectories = ['spec/javascripts', 'spec/frontend'];
+
+if (fs.existsSync('ee')) {
+  sourceDirectories.forEach(dir => {
+    sourceDirectories.push(`ee/${dir}`);
+  });
+
+  testDirectories.forEach(dir => {
+    testDirectories.push(`ee/${dir}`);
+  });
+}
+
+let numSourceFiles = 0;
+let numTestFiles = 0;
+
+const isVerbose = process.argv.some(arg => arg === '-v');
+
+const countSourceFiles = path =>
+  forEachFileIn(path, fileName => {
+    if (fileName.endsWith('.vue') || fileName.endsWith('.js')) {
+      if (isVerbose) {
+        console.log(`source file: ${fileName}`);
+      }
+
+      numSourceFiles += 1;
+    }
+  });
+
+const countTestFiles = path =>
+  forEachFileIn(path, fileName => {
+    if (fileName.endsWith('_spec.js')) {
+      if (isVerbose) {
+        console.log(`test file: ${fileName}`);
+      }
+
+      numTestFiles += 1;
+    }
+  });
+
+function forEachFileIn(dirPath, callback) {
+  fs.readdir(dirPath, (err, files) => {
+    if (err) {
+      console.error(err);
+    }
+
+    if (!files) {
+      return;
+    }
+
+    files.forEach(fileName => {
+      const absolutePath = path.join(dirPath, fileName);
+      const stats = fs.statSync(absolutePath);
+      if (stats.isFile()) {
+        callback(absolutePath);
+      } else if (stats.isDirectory()) {
+        forEachFileIn(absolutePath, callback);
+      }
+    });
+  });
+}
+
+console.log(`Source directories: ${sourceDirectories.join(', ')}`);
+console.log(`Test directories: ${testDirectories.join(', ')}`);
+
+sourceDirectories.forEach(countSourceFiles);
+testDirectories.forEach(countTestFiles);
+
+process.on('exit', () => {
+  console.log(`Source files: ${numSourceFiles}`);
+  console.log(`Test files: ${numTestFiles}`);
+  console.log(`Coverage: ${((100 * numTestFiles) / numSourceFiles).toFixed(2)}%`);
+});
diff --git a/scripts/gather-test-memory-data b/scripts/gather-test-memory-data
new file mode 100755
index 00000000000..9992a83e6a6
--- /dev/null
+++ b/scripts/gather-test-memory-data
@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+
+def join_csv_files(output_path, input_paths)
+  return if input_paths.empty?
+
+  input_csvs = input_paths.map do |input_path|
+    CSV.read(input_path, headers: true)
+  end
+
+  CSV.open(output_path, "w", headers: input_csvs.first.headers, write_headers: true) do |output_csv|
+    input_csvs.each do |input_csv|
+      input_csv.each do |line|
+        output_csv << line
+      end
+    end
+  end
+end
+
+join_csv_files('tmp/memory_test/report.csv', Dir['tmp/memory_test/*.csv'].sort)
diff --git a/scripts/insert-rspec-profiling-data b/scripts/insert-rspec-profiling-data
index b34379764e0..88c9d8c12b1 100755
--- a/scripts/insert-rspec-profiling-data
+++ b/scripts/insert-rspec-profiling-data
@@ -14,10 +14,6 @@ module RspecProfiling
         Result.establish_connection(results_url)
       end
 
-      def prepared?
-        connection.data_source_exists?(table)
-      end
-
       def results_url
         ENV['RSPEC_PROFILING_POSTGRES_URL']
       end
diff --git a/scripts/lint-changelog-yaml b/scripts/lint-changelog-yaml
deleted file mode 100755
index 06d502c4676..00000000000
--- a/scripts/lint-changelog-yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'yaml'
-
-invalid_changelogs = Dir['changelogs/**/*'].reject do |changelog|
-  next true if changelog =~ /((README|archive)\.md|unreleased(-ee)?)$/
-  next false unless changelog.end_with?('.yml')
-
-  begin
-    YAML.load_file(changelog)
-  rescue => exception
-    puts exception
-  end
-end
-
-if invalid_changelogs.any?
-  puts
-  puts "Invalid changelogs found!\n"
-  puts invalid_changelogs.sort
-  exit 1
-else
-  puts "All changelogs are valid YAML.\n"
-  exit 0
-end
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index 2055ce7f09d..8c9b8b9fb02 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -45,7 +45,7 @@ then
   then
     echo
     echo '  ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2
-    echo '  https://docs.gitlab.com/ee/development/writing_documentation.html#location-and-naming-documents'
+    echo '  https://docs.gitlab.com/ee/development/documentation/styleguide.html#working-with-directories-and-files'
     echo
     exit 1
   fi
@@ -55,7 +55,7 @@ then
   then
     echo
     echo '  ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2
-    echo '  https://docs.gitlab.com/ee/development/writing_documentation.html#location-and-naming-documents'
+    echo '  https://docs.gitlab.com/ee/development/documentation/styleguide.html#working-with-directories-and-files'
     echo
     exit 1
   fi
diff --git a/scripts/lint-rugged b/scripts/lint-rugged
index 9466c62a415..1b3fb54f70b 100755
--- a/scripts/lint-rugged
+++ b/scripts/lint-rugged
@@ -1,6 +1,9 @@
 #!/usr/bin/env ruby
 
 ALLOWED = [
+  # https://gitlab.com/gitlab-org/gitaly/issues/760
+  'lib/elasticsearch/git/repository.rb',
+
   # Needed to handle repositories that are not in any storage
   'lib/gitlab/bare_repository_import/repository.rb',
 
@@ -10,7 +13,15 @@ ALLOWED = [
   # Reverted Rugged calls due to Gitaly atop NFS performance
   # See https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code.
   'lib/gitlab/git/rugged_impl/',
-  'lib/gitlab/gitaly_client/storage_settings.rb'
+  'lib/gitlab/gitaly_client/storage_settings.rb',
+
+  # Needed for logging
+  'config/initializers/peek.rb',
+  'config/initializers/lograge.rb',
+  'lib/gitlab/grape_logging/loggers/perf_logger.rb',
+  'lib/gitlab/instrumentation_helper.rb',
+  'lib/gitlab/rugged_instrumentation.rb',
+  'lib/peek/views/rugged.rb'
 ].freeze
 
 rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines
diff --git a/scripts/merge-html-reports b/scripts/merge-html-reports
new file mode 100755
index 00000000000..7d1e15186c8
--- /dev/null
+++ b/scripts/merge-html-reports
@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+
+main_report_file = ARGV.shift
+unless main_report_file
+  puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+  exit 1
+end
+
+base_artifact_url = ARGV.shift
+unless base_artifact_url
+  puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+  exit 1
+end
+
+# Create the base report with empty body tag
+new_report = Nokogiri::HTML.parse(File.read(ARGV[0]))
+new_report.at_css('body').remove
+empty_body = Nokogiri::XML::Node.new('body', new_report)
+new_report.at_css('head').add_next_sibling(empty_body)
+
+ARGV.each do |report_file|
+  report = Nokogiri::HTML.parse(File.read(report_file))
+
+  report.css('a').each do |link|
+    link_suffix = link['href'].slice(19..-1)
+    link['href'] = base_artifact_url + link_suffix
+  end
+
+  header = report.css('div #rspec-header')
+  tests = report.css('dt[id^="example_group_"]')
+
+  tests.each do |test|
+    title = test.parent
+    group = title.parent
+    script = title.css('script')
+
+    if script.inner_html.include? 'makeYellow'
+      test.remove_class('passed')
+      test.add_class('not_implemented')
+
+      group.remove_class('passed')
+      group.add_class('not_implemented')
+      header.add_class('not_implemented')
+
+      script.remove
+      test.next_sibling.remove
+      test.next_sibling.remove
+
+    elsif script.inner_html.include? 'makeRed'
+      test.remove_class('passed')
+      test.add_class('failed')
+
+      group.remove_class('passed')
+      group.add_class('failed')
+      header.add_class('failed')
+
+      script.remove
+      test.next_sibling.remove
+      test.next_sibling.remove
+    end
+  end
+
+  duration = report.at_css('p#duration')
+  totals = report.at_css('p#totals')
+
+  duration_script = report.css('div.results script')[-2]
+  totals_script = report.css('div.results script')[-1]
+
+  duration_text = duration_script.text.slice(49..-3)
+  totals_text = totals_script.text.slice(47..-3)
+
+  duration.inner_html = duration_text
+  totals.inner_html = totals_text
+
+  duration_script.remove
+  totals_script.remove
+
+  # Add the new result after the last one to keep the test order
+  new_report.css('body')[-1].add_next_sibling(report.at_css('body'))
+end
+
+File.write(main_report_file, new_report)
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 2bf654b1e24..4935c1342a3 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -52,7 +52,7 @@ function delete() {
 function get_pod() {
   local app_name="${1}"
   local status="${2-Running}"
-  get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name"
+  get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
   echoinfo "Waiting till '${app_name}' pod is ready" true
   echoinfo "Running '${get_pod_cmd}'"
 
@@ -69,7 +69,6 @@ function get_pod() {
       break
     fi
 
-    printf "."
     let "elapsed_seconds+=interval"
     sleep ${interval}
   done
@@ -132,13 +131,14 @@ function install_external_dns() {
     echoinfo "Installing external-dns Helm chart"
     helm repo update
     # Default requested: CPU => 0, memory => 0
-    helm install stable/external-dns \
+    helm install stable/external-dns --version '^2.2.1' \
       -n "${release_name}" \
       --namespace "${KUBE_NAMESPACE}" \
       --set provider="aws" \
-      --set aws.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \
-      --set aws.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \
+      --set aws.credentials.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \
+      --set aws.credentials.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \
       --set aws.zoneType="public" \
+      --set aws.batchChangeSize=400 \
       --set domainFilters[0]="${domain}" \
       --set txtOwnerId="${KUBE_NAMESPACE}" \
       --set rbac.create="true" \
@@ -189,18 +189,12 @@ function deploy() {
   gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
   gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}"
 
-  # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
-  if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then
-    echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
-    delete
-  fi
-
   create_application_secret
 
 HELM_CMD=$(cat << EOF
   helm upgrade --install \
-    --wait \
-    --timeout 600 \
+    --atomic \
+    --timeout 900 \
     --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
     --set global.appConfig.enableUsagePing=false \
     --set global.imagePullPolicy=Always \
@@ -346,32 +340,6 @@ function display_deployment_debug() {
   fi
 }
 
-function wait_for_review_app_to_be_accessible() {
-  echoinfo "Waiting for the Review App at ${CI_ENVIRONMENT_URL} to be accessible..." true
-
-  local interval=5
-  local elapsed_seconds=0
-  local max_seconds=$((2 * 60))
-  while true; do
-    local review_app_http_code
-    review_app_http_code=$(curl --silent --output /dev/null --max-time 5 --write-out "%{http_code}" "${CI_ENVIRONMENT_URL}/users/sign_in")
-    if [[ "${review_app_http_code}" -eq "200" ]] || [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
-      break
-    fi
-
-    printf "."
-    let "elapsed_seconds+=interval"
-    sleep ${interval}
-  done
-
-  if [[ "${review_app_http_code}" -eq "200" ]]; then
-    echoinfo "The Review App at ${CI_ENVIRONMENT_URL} is ready after ${elapsed_seconds} seconds!"
-  else
-    echoerr "The Review App at ${CI_ENVIRONMENT_URL} isn't ready after ${max_seconds} seconds of polling..."
-    exit 1
-  fi
-}
-
 function add_license() {
   if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
 
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 642c50ec0a8..6fd64fbf9da 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 # We don't have auto-loading here
+require_relative '../lib/gitlab'
 require_relative '../lib/gitlab/popen'
 require_relative '../lib/gitlab/popen/runner'
 
@@ -36,6 +37,10 @@ tasks = [
   %w[scripts/lint-rugged]
 ]
 
+if Gitlab.ee?
+  tasks.unshift(%w[ruby -rbundler/setup scripts/ee_specific_check/ee_specific_check.rb])
+end
+
 static_analysis = Gitlab::Popen::Runner.new
 
 static_analysis.run(tasks) do |cmd, &run|
diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs
index 0fc8f6fbd4b..83841512f1c 100755
--- a/scripts/trigger-build-docs
+++ b/scripts/trigger-build-docs
@@ -13,7 +13,7 @@ end
 #
 # The remote docs project
 #
-GITLAB_DOCS_REPO = 'gitlab-com/gitlab-docs'.freeze
+GITLAB_DOCS_REPO = 'gitlab-org/gitlab-docs'.freeze
 
 #
 # Truncate the remote docs branch name otherwise we hit the filesystem
@@ -31,7 +31,7 @@ end
 # to avoid race conditions, since a triggered pipeline will also run right
 # after the branch creation. This only happens the very first time a branch
 # is created and will be skipped in subsequent runs. Read more in
-# https://gitlab.com/gitlab-com/gitlab-docs/issues/154.
+# https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
 #
 def create_remote_branch
   Gitlab.create_branch(GITLAB_DOCS_REPO, docs_branch, 'master')
@@ -81,7 +81,7 @@ def slug
 end
 
 #
-# Overriding vars in https://gitlab.com/gitlab-com/gitlab-docs/blob/master/.gitlab-ci.yml
+# Overriding vars in https://gitlab.com/gitlab-org/gitlab-docs/blob/master/.gitlab-ci.yml
 #
 def param_name
   "BRANCH_#{slug.upcase}"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 4a6567b8a62..f0f08e2e1c5 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -29,6 +29,8 @@ function setup_db() {
     if [ "$GITLAB_DATABASE" = "mysql" ]; then
         bundle exec rake add_limits_mysql
     fi
+
+    bundle exec rake gitlab:db:setup_ee
 }
 
 function install_api_client_dependencies_with_apk() {