summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rw-r--r--scripts/create_mysql_user.sh7
-rwxr-xr-xscripts/frontend/file_test_coverage.js88
-rwxr-xr-xscripts/gather-test-memory-data21
-rwxr-xr-xscripts/insert-rspec-profiling-data4
-rwxr-xr-xscripts/lint-changelog-yaml24
-rwxr-xr-xscripts/lint-doc.sh4
-rwxr-xr-xscripts/lint-rugged13
-rwxr-xr-xscripts/merge-html-reports84
-rwxr-xr-xscripts/review_apps/review-apps.sh46
-rwxr-xr-xscripts/static-analysis5
-rwxr-xr-xscripts/trigger-build-docs6
-rw-r--r--scripts/utils.sh2
12 files changed, 224 insertions, 80 deletions
diff --git a/scripts/create_mysql_user.sh b/scripts/create_mysql_user.sh
deleted file mode 100644
index 35f68c581f3..00000000000
--- a/scripts/create_mysql_user.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-mysql --user=root --host=mysql <<EOF
-CREATE USER IF NOT EXISTS 'gitlab'@'%';
-GRANT ALL PRIVILEGES ON gitlabhq_test.* TO 'gitlab'@'%';
-FLUSH PRIVILEGES;
-EOF
diff --git a/scripts/frontend/file_test_coverage.js b/scripts/frontend/file_test_coverage.js
new file mode 100755
index 00000000000..7d1eb45d4bc
--- /dev/null
+++ b/scripts/frontend/file_test_coverage.js
@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+
+/**
+ * Counts the number of frontend test files and compares them against the number of application files.
+ *
+ * Example output:
+ *
+ * Source files: 1551
+ * Test files: 716
+ * Coverage: 46.16%
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const sourceDirectories = ['app/assets/javascripts'];
+const testDirectories = ['spec/javascripts', 'spec/frontend'];
+
+if (fs.existsSync('ee')) {
+ sourceDirectories.forEach(dir => {
+ sourceDirectories.push(`ee/${dir}`);
+ });
+
+ testDirectories.forEach(dir => {
+ testDirectories.push(`ee/${dir}`);
+ });
+}
+
+let numSourceFiles = 0;
+let numTestFiles = 0;
+
+const isVerbose = process.argv.some(arg => arg === '-v');
+
+const countSourceFiles = path =>
+ forEachFileIn(path, fileName => {
+ if (fileName.endsWith('.vue') || fileName.endsWith('.js')) {
+ if (isVerbose) {
+ console.log(`source file: ${fileName}`);
+ }
+
+ numSourceFiles += 1;
+ }
+ });
+
+const countTestFiles = path =>
+ forEachFileIn(path, fileName => {
+ if (fileName.endsWith('_spec.js')) {
+ if (isVerbose) {
+ console.log(`test file: ${fileName}`);
+ }
+
+ numTestFiles += 1;
+ }
+ });
+
+function forEachFileIn(dirPath, callback) {
+ fs.readdir(dirPath, (err, files) => {
+ if (err) {
+ console.error(err);
+ }
+
+ if (!files) {
+ return;
+ }
+
+ files.forEach(fileName => {
+ const absolutePath = path.join(dirPath, fileName);
+ const stats = fs.statSync(absolutePath);
+ if (stats.isFile()) {
+ callback(absolutePath);
+ } else if (stats.isDirectory()) {
+ forEachFileIn(absolutePath, callback);
+ }
+ });
+ });
+}
+
+console.log(`Source directories: ${sourceDirectories.join(', ')}`);
+console.log(`Test directories: ${testDirectories.join(', ')}`);
+
+sourceDirectories.forEach(countSourceFiles);
+testDirectories.forEach(countTestFiles);
+
+process.on('exit', () => {
+ console.log(`Source files: ${numSourceFiles}`);
+ console.log(`Test files: ${numTestFiles}`);
+ console.log(`Coverage: ${((100 * numTestFiles) / numSourceFiles).toFixed(2)}%`);
+});
diff --git a/scripts/gather-test-memory-data b/scripts/gather-test-memory-data
new file mode 100755
index 00000000000..9992a83e6a6
--- /dev/null
+++ b/scripts/gather-test-memory-data
@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+
+def join_csv_files(output_path, input_paths)
+ return if input_paths.empty?
+
+ input_csvs = input_paths.map do |input_path|
+ CSV.read(input_path, headers: true)
+ end
+
+ CSV.open(output_path, "w", headers: input_csvs.first.headers, write_headers: true) do |output_csv|
+ input_csvs.each do |input_csv|
+ input_csv.each do |line|
+ output_csv << line
+ end
+ end
+ end
+end
+
+join_csv_files('tmp/memory_test/report.csv', Dir['tmp/memory_test/*.csv'].sort)
diff --git a/scripts/insert-rspec-profiling-data b/scripts/insert-rspec-profiling-data
index b34379764e0..88c9d8c12b1 100755
--- a/scripts/insert-rspec-profiling-data
+++ b/scripts/insert-rspec-profiling-data
@@ -14,10 +14,6 @@ module RspecProfiling
Result.establish_connection(results_url)
end
- def prepared?
- connection.data_source_exists?(table)
- end
-
def results_url
ENV['RSPEC_PROFILING_POSTGRES_URL']
end
diff --git a/scripts/lint-changelog-yaml b/scripts/lint-changelog-yaml
deleted file mode 100755
index 06d502c4676..00000000000
--- a/scripts/lint-changelog-yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'yaml'
-
-invalid_changelogs = Dir['changelogs/**/*'].reject do |changelog|
- next true if changelog =~ /((README|archive)\.md|unreleased(-ee)?)$/
- next false unless changelog.end_with?('.yml')
-
- begin
- YAML.load_file(changelog)
- rescue => exception
- puts exception
- end
-end
-
-if invalid_changelogs.any?
- puts
- puts "Invalid changelogs found!\n"
- puts invalid_changelogs.sort
- exit 1
-else
- puts "All changelogs are valid YAML.\n"
- exit 0
-end
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index 2055ce7f09d..8c9b8b9fb02 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -45,7 +45,7 @@ then
then
echo
echo ' ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2
- echo ' https://docs.gitlab.com/ee/development/writing_documentation.html#location-and-naming-documents'
+ echo ' https://docs.gitlab.com/ee/development/documentation/styleguide.html#working-with-directories-and-files'
echo
exit 1
fi
@@ -55,7 +55,7 @@ then
then
echo
echo ' ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2
- echo ' https://docs.gitlab.com/ee/development/writing_documentation.html#location-and-naming-documents'
+ echo ' https://docs.gitlab.com/ee/development/documentation/styleguide.html#working-with-directories-and-files'
echo
exit 1
fi
diff --git a/scripts/lint-rugged b/scripts/lint-rugged
index 9466c62a415..1b3fb54f70b 100755
--- a/scripts/lint-rugged
+++ b/scripts/lint-rugged
@@ -1,6 +1,9 @@
#!/usr/bin/env ruby
ALLOWED = [
+ # https://gitlab.com/gitlab-org/gitaly/issues/760
+ 'lib/elasticsearch/git/repository.rb',
+
# Needed to handle repositories that are not in any storage
'lib/gitlab/bare_repository_import/repository.rb',
@@ -10,7 +13,15 @@ ALLOWED = [
# Reverted Rugged calls due to Gitaly atop NFS performance
# See https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code.
'lib/gitlab/git/rugged_impl/',
- 'lib/gitlab/gitaly_client/storage_settings.rb'
+ 'lib/gitlab/gitaly_client/storage_settings.rb',
+
+ # Needed for logging
+ 'config/initializers/peek.rb',
+ 'config/initializers/lograge.rb',
+ 'lib/gitlab/grape_logging/loggers/perf_logger.rb',
+ 'lib/gitlab/instrumentation_helper.rb',
+ 'lib/gitlab/rugged_instrumentation.rb',
+ 'lib/peek/views/rugged.rb'
].freeze
rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines
diff --git a/scripts/merge-html-reports b/scripts/merge-html-reports
new file mode 100755
index 00000000000..7d1e15186c8
--- /dev/null
+++ b/scripts/merge-html-reports
@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+
+main_report_file = ARGV.shift
+unless main_report_file
+ puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+ exit 1
+end
+
+base_artifact_url = ARGV.shift
+unless base_artifact_url
+ puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+ exit 1
+end
+
+# Create the base report with empty body tag
+new_report = Nokogiri::HTML.parse(File.read(ARGV[0]))
+new_report.at_css('body').remove
+empty_body = Nokogiri::XML::Node.new('body', new_report)
+new_report.at_css('head').add_next_sibling(empty_body)
+
+ARGV.each do |report_file|
+ report = Nokogiri::HTML.parse(File.read(report_file))
+
+ report.css('a').each do |link|
+ link_suffix = link['href'].slice(19..-1)
+ link['href'] = base_artifact_url + link_suffix
+ end
+
+ header = report.css('div #rspec-header')
+ tests = report.css('dt[id^="example_group_"]')
+
+ tests.each do |test|
+ title = test.parent
+ group = title.parent
+ script = title.css('script')
+
+ if script.inner_html.include? 'makeYellow'
+ test.remove_class('passed')
+ test.add_class('not_implemented')
+
+ group.remove_class('passed')
+ group.add_class('not_implemented')
+ header.add_class('not_implemented')
+
+ script.remove
+ test.next_sibling.remove
+ test.next_sibling.remove
+
+ elsif script.inner_html.include? 'makeRed'
+ test.remove_class('passed')
+ test.add_class('failed')
+
+ group.remove_class('passed')
+ group.add_class('failed')
+ header.add_class('failed')
+
+ script.remove
+ test.next_sibling.remove
+ test.next_sibling.remove
+ end
+ end
+
+ duration = report.at_css('p#duration')
+ totals = report.at_css('p#totals')
+
+ duration_script = report.css('div.results script')[-2]
+ totals_script = report.css('div.results script')[-1]
+
+ duration_text = duration_script.text.slice(49..-3)
+ totals_text = totals_script.text.slice(47..-3)
+
+ duration.inner_html = duration_text
+ totals.inner_html = totals_text
+
+ duration_script.remove
+ totals_script.remove
+
+ # Add the new result after the last one to keep the test order
+ new_report.css('body')[-1].add_next_sibling(report.at_css('body'))
+end
+
+File.write(main_report_file, new_report)
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 2bf654b1e24..4935c1342a3 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -52,7 +52,7 @@ function delete() {
function get_pod() {
local app_name="${1}"
local status="${2-Running}"
- get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name"
+ get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
echoinfo "Waiting till '${app_name}' pod is ready" true
echoinfo "Running '${get_pod_cmd}'"
@@ -69,7 +69,6 @@ function get_pod() {
break
fi
- printf "."
let "elapsed_seconds+=interval"
sleep ${interval}
done
@@ -132,13 +131,14 @@ function install_external_dns() {
echoinfo "Installing external-dns Helm chart"
helm repo update
# Default requested: CPU => 0, memory => 0
- helm install stable/external-dns \
+ helm install stable/external-dns --version '^2.2.1' \
-n "${release_name}" \
--namespace "${KUBE_NAMESPACE}" \
--set provider="aws" \
- --set aws.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \
- --set aws.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \
+ --set aws.credentials.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \
+ --set aws.credentials.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \
--set aws.zoneType="public" \
+ --set aws.batchChangeSize=400 \
--set domainFilters[0]="${domain}" \
--set txtOwnerId="${KUBE_NAMESPACE}" \
--set rbac.create="true" \
@@ -189,18 +189,12 @@ function deploy() {
gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}"
- # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
- if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then
- echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
- delete
- fi
-
create_application_secret
HELM_CMD=$(cat << EOF
helm upgrade --install \
- --wait \
- --timeout 600 \
+ --atomic \
+ --timeout 900 \
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
--set global.appConfig.enableUsagePing=false \
--set global.imagePullPolicy=Always \
@@ -346,32 +340,6 @@ function display_deployment_debug() {
fi
}
-function wait_for_review_app_to_be_accessible() {
- echoinfo "Waiting for the Review App at ${CI_ENVIRONMENT_URL} to be accessible..." true
-
- local interval=5
- local elapsed_seconds=0
- local max_seconds=$((2 * 60))
- while true; do
- local review_app_http_code
- review_app_http_code=$(curl --silent --output /dev/null --max-time 5 --write-out "%{http_code}" "${CI_ENVIRONMENT_URL}/users/sign_in")
- if [[ "${review_app_http_code}" -eq "200" ]] || [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
- break
- fi
-
- printf "."
- let "elapsed_seconds+=interval"
- sleep ${interval}
- done
-
- if [[ "${review_app_http_code}" -eq "200" ]]; then
- echoinfo "The Review App at ${CI_ENVIRONMENT_URL} is ready after ${elapsed_seconds} seconds!"
- else
- echoerr "The Review App at ${CI_ENVIRONMENT_URL} isn't ready after ${max_seconds} seconds of polling..."
- exit 1
- fi
-}
-
function add_license() {
if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 642c50ec0a8..6fd64fbf9da 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -1,6 +1,7 @@
#!/usr/bin/env ruby
# We don't have auto-loading here
+require_relative '../lib/gitlab'
require_relative '../lib/gitlab/popen'
require_relative '../lib/gitlab/popen/runner'
@@ -36,6 +37,10 @@ tasks = [
%w[scripts/lint-rugged]
]
+if Gitlab.ee?
+ tasks.unshift(%w[ruby -rbundler/setup scripts/ee_specific_check/ee_specific_check.rb])
+end
+
static_analysis = Gitlab::Popen::Runner.new
static_analysis.run(tasks) do |cmd, &run|
diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs
index 0fc8f6fbd4b..83841512f1c 100755
--- a/scripts/trigger-build-docs
+++ b/scripts/trigger-build-docs
@@ -13,7 +13,7 @@ end
#
# The remote docs project
#
-GITLAB_DOCS_REPO = 'gitlab-com/gitlab-docs'.freeze
+GITLAB_DOCS_REPO = 'gitlab-org/gitlab-docs'.freeze
#
# Truncate the remote docs branch name otherwise we hit the filesystem
@@ -31,7 +31,7 @@ end
# to avoid race conditions, since a triggered pipeline will also run right
# after the branch creation. This only happens the very first time a branch
# is created and will be skipped in subsequent runs. Read more in
-# https://gitlab.com/gitlab-com/gitlab-docs/issues/154.
+# https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
#
def create_remote_branch
Gitlab.create_branch(GITLAB_DOCS_REPO, docs_branch, 'master')
@@ -81,7 +81,7 @@ def slug
end
#
-# Overriding vars in https://gitlab.com/gitlab-com/gitlab-docs/blob/master/.gitlab-ci.yml
+# Overriding vars in https://gitlab.com/gitlab-org/gitlab-docs/blob/master/.gitlab-ci.yml
#
def param_name
"BRANCH_#{slug.upcase}"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 4a6567b8a62..f0f08e2e1c5 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -29,6 +29,8 @@ function setup_db() {
if [ "$GITLAB_DATABASE" = "mysql" ]; then
bundle exec rake add_limits_mysql
fi
+
+ bundle exec rake gitlab:db:setup_ee
}
function install_api_client_dependencies_with_apk() {