summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-10-20 08:43:02 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2021-10-20 08:43:02 +0000
commitd9ab72d6080f594d0b3cae15f14b3ef2c6c638cb (patch)
tree2341ef426af70ad1e289c38036737e04b0aa5007 /scripts
parentd6e514dd13db8947884cd58fe2a9c2a063400a9b (diff)
downloadgitlab-ce-d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb.tar.gz
Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/frontend/start_storybook.sh18
-rwxr-xr-xscripts/frontend/startup_css/startup_css_changed.sh8
-rwxr-xr-xscripts/lint-doc.sh18
-rw-r--r--scripts/prepare_build.sh12
-rwxr-xr-xscripts/rails-update-bisect39
-rwxr-xr-xscripts/review_apps/automated_cleanup.rb49
-rw-r--r--scripts/rspec_helpers.sh14
-rwxr-xr-xscripts/rubocop-max-files-in-cache-check27
-rwxr-xr-xscripts/static-analysis5
-rw-r--r--scripts/utils.sh34
-rwxr-xr-xscripts/verify-tff-mapping8
11 files changed, 194 insertions, 38 deletions
diff --git a/scripts/frontend/start_storybook.sh b/scripts/frontend/start_storybook.sh
new file mode 100755
index 00000000000..7ae0a21b15b
--- /dev/null
+++ b/scripts/frontend/start_storybook.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+bold=$(tput bold)
+normal=$(tput sgr0)
+
+echo -e "Storybook provides a mock server that allows creating stories for components that make HTTP requests."
+echo -e "${bold}Storybook will fail to start if it can’t find the fixtures used by the mock server.${normal}\n"
+read -rp "Would you like to generate/update the frontend fixtures used by the mock server (y/N)? " answer
+
+if [[ "$answer" =~ ^(Y|y)$ ]] ; then
+ bundle exec rake frontend:mock_server_fixtures
+fi
+
+if ! [[ -d storybook/node_modules ]] ; then
+ yarn storybook:install
+fi
+
+yarn --cwd ./storybook start
diff --git a/scripts/frontend/startup_css/startup_css_changed.sh b/scripts/frontend/startup_css/startup_css_changed.sh
index 2713d752974..db6fb575d1d 100755
--- a/scripts/frontend/startup_css/startup_css_changed.sh
+++ b/scripts/frontend/startup_css/startup_css_changed.sh
@@ -7,7 +7,13 @@ echo ""
echo "https://gitlab.com/gitlab-org/gitlab/-/issues/331812"
echo "-----------------------------------------------------------"
-startup_glob="*stylesheets/startup*"
+startup_glob="app/assets/stylesheets/startup*"
+
+if ! [ "$FOSS_ONLY" ]
+then
+ startup_glob="*${startup_glob}"
+fi
+
echo "Staging changes to '${startup_glob}' so we can check for untracked files..."
git add "${startup_glob}"
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index 0157f0c0812..e99b8a47301 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -67,7 +67,7 @@ fi
# Do not use 'README.md', instead use 'index.md'
# Number of 'README.md's as of 2021-08-17
-NUMBER_READMES=13
+NUMBER_READMES=0
FIND_READMES=$(find doc/ -name "README.md" | wc -l)
echo '=> Checking for new README.md files...'
echo
@@ -81,6 +81,22 @@ then
((ERRORCODE++))
fi
+# Do not use dashes (-) in directory names, use underscores (_) instead.
+# Number of directories with dashes as of 2021-09-17
+NUMBER_DASHES=2
+FIND_DASHES=$(find doc -type d -name "*-*" | wc -l)
+echo '=> Checking for directory names containing dashes...'
+echo
+if [ ${FIND_DASHES} -ne $NUMBER_DASHES ]
+then
+ echo
+ echo ' ✖ ERROR: The number of directory names containing dashes has changed. Use underscores instead of dashes for the directory names.' >&2
+ echo ' ✖ If removing a directory containing dashes, update NUMBER_DASHES in lint-doc.sh.' >&2
+ echo ' https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#work-with-directories-and-files'
+ echo
+ ((ERRORCODE++))
+fi
+
# Run Vale and Markdownlint only on changed files. Only works on merged results
# pipelines, so first checks if a merged results CI variable is present. If not present,
# runs test on all files.
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index f3b9ac56082..4b528696322 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -38,18 +38,6 @@ sed -i 's|url:.*$|url: redis://redis:6379|g' config/cable.yml
cp config/resque.yml.example config/resque.yml
sed -i 's|url:.*$|url: redis://redis:6379|g' config/resque.yml
-cp config/redis.cache.yml.example config/redis.cache.yml
-sed -i 's|url:.*$|url: redis://redis:6379/10|g' config/redis.cache.yml
-
-cp config/redis.queues.yml.example config/redis.queues.yml
-sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml
-
-cp config/redis.shared_state.yml.example config/redis.shared_state.yml
-sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
-
-cp config/redis.trace_chunks.yml.example config/redis.trace_chunks.yml
-sed -i 's|url:.*$|url: redis://redis:6379/13|g' config/redis.trace_chunks.yml
-
if [ "$SETUP_DB" != "false" ]; then
setup_db
elif getent hosts postgres; then
diff --git a/scripts/rails-update-bisect b/scripts/rails-update-bisect
new file mode 100755
index 00000000000..77a01e2dadc
--- /dev/null
+++ b/scripts/rails-update-bisect
@@ -0,0 +1,39 @@
+#!/bin/bash
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+GITLAB_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Because this script is intended to be run with `git bisect run`,
+# we are returning `-1` status code to alert `git bisect` of failures.
+#
+# See: https://git-scm.com/docs/git-bisect#_bisect_run
+#
+abort_bisect () {
+ exit -1
+}
+
+if [ $# -eq 0 ]; then
+ echo "No arguments supplied. Please provide spec file(s) as first argument(s)"
+ abort_bisect
+fi
+
+[[ -z "${RAILS_FOLDER}" ]] && { echo >&2 "RAILS_FOLDER env variable is not set"; abort_bisect; }
+
+if ! grep -q -E "gem 'rails'.+RAILS_VERSION.+RAILS_FOLDER" $GITLAB_DIR/Gemfile; then
+ echo "Gemfile is not modified"
+ echo "Please alter the gem 'rails' line in Gemfile with:"
+ echo "gem 'rails', ENV['RAILS_VERSION'], path: ENV['RAILS_FOLDER']"
+ abort_bisect
+fi
+
+export RAILS_VERSION=$(cat $RAILS_FOLDER/RAILS_VERSION)
+
+cd $GITLAB_DIR && \
+echo "Updating dependencies... this could take a while." && \
+bundle update rails --quiet
+
+test $? -eq 0 || { echo >&2 "bundle update has failed"; abort_bisect; }
+
+# Stop spring if it's installed
+command -v spring >/dev/null 2>&1 && spring stop
+
+bin/rspec $@
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index 90dc0fd418e..e6efbca9e86 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -8,6 +8,10 @@ class AutomatedCleanup
attr_reader :project_path, :gitlab_token
DEPLOYMENTS_PER_PAGE = 100
+ ENVIRONMENT_PREFIX = {
+ review_app: 'review/',
+ docs_review_app: 'review-docs/'
+ }.freeze
IGNORED_HELM_ERRORS = [
'transport is closing',
'error upgrading connection',
@@ -62,13 +66,14 @@ class AutomatedCleanup
releases_to_delete = []
+ # Delete environments via deployments
gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
break if Time.parse(deployment.created_at) < deployments_look_back_threshold
environment = deployment.environment
next unless environment
- next unless environment.name.start_with?('review/')
+ next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:review_app])
next if checked_environments.include?(environment.slug)
last_deploy = deployment.created_at
@@ -92,6 +97,10 @@ class AutomatedCleanup
checked_environments << environment.slug
end
+ delete_stopped_environments(environment_type: :review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) do |environment|
+ releases_to_delete << Tooling::Helm3Client::Release.new(environment.slug, 1, environment.updated_at, nil, nil, review_apps_namespace)
+ end
+
delete_helm_releases(releases_to_delete)
end
@@ -102,14 +111,12 @@ class AutomatedCleanup
stop_threshold = threshold_time(days: days_for_stop)
delete_threshold = threshold_time(days: days_for_delete)
- max_delete_count = 1000
- delete_count = 0
-
+ # Delete environments via deployments
gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
environment = deployment.environment
next unless environment
- next unless environment.name.start_with?('review-docs/')
+ next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:docs_review_app])
next if checked_environments.include?(environment.slug)
last_deploy = deployment.created_at
@@ -120,15 +127,12 @@ class AutomatedCleanup
stop_environment(environment, deployment) if environment_state && environment_state != 'stopped'
end
- if deployed_at < delete_threshold
- delete_environment(environment, deployment)
- delete_count += 1
-
- break if delete_count > max_delete_count
- end
+ delete_environment(environment, deployment) if deployed_at < delete_threshold
checked_environments << environment.slug
end
+
+ delete_stopped_environments(environment_type: :docs_review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold)
end
def perform_helm_releases_cleanup!(days:)
@@ -171,8 +175,9 @@ class AutomatedCleanup
nil
end
- def delete_environment(environment, deployment)
- print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'deleting')
+ def delete_environment(environment, deployment = nil)
+ release_date = deployment ? deployment.created_at : environment.updated_at
+ print_release_state(subject: 'Review app', release_name: environment.slug, release_date: release_date, action: 'deleting')
gitlab.delete_environment(project_path, environment.id)
rescue Gitlab::Error::Forbidden
@@ -187,6 +192,24 @@ class AutomatedCleanup
puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
end
+ def delete_stopped_environments(environment_type:, checked_environments:, last_updated_threshold:)
+ gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment|
+ next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type)
+
+ yield environment if delete_environment(environment)
+
+ checked_environments << environment.slug
+ end
+ end
+
+ def skip_environment?(environment:, checked_environments:, last_updated_threshold:, environment_type:)
+ return true unless environment.name.start_with?(ENVIRONMENT_PREFIX[environment_type])
+ return true if checked_environments.include?(environment.slug)
+ return true if Time.parse(environment.updated_at) > last_updated_threshold
+
+ false
+ end
+
def helm_releases
args = ['--all', '--date']
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 797d9188f81..accc52a7ece 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -98,7 +98,7 @@ function rspec_simple_job() {
}
function rspec_db_library_code() {
- local db_files="spec/lib/gitlab/database/ spec/support/helpers/database/"
+ local db_files="spec/lib/gitlab/database/"
rspec_simple_job "-- ${db_files}"
}
@@ -109,14 +109,18 @@ function rspec_paralellized_job() {
local test_level="${job_name[1]}"
local report_name=$(echo "${CI_JOB_NAME}" | sed -E 's|[/ ]|_|g') # e.g. 'rspec unit pg12 1/24' would become 'rspec_unit_pg12_1_24'
local rspec_opts="${1}"
- local spec_folder_prefix=""
+ local spec_folder_prefixes=""
if [[ "${test_tool}" =~ "-ee" ]]; then
- spec_folder_prefix="ee/"
+ spec_folder_prefixes="'ee/'"
fi
if [[ "${test_tool}" =~ "-jh" ]]; then
- spec_folder_prefix="jh/"
+ spec_folder_prefixes="'jh/'"
+ fi
+
+ if [[ "${test_tool}" =~ "-all" ]]; then
+ spec_folder_prefixes="['', 'ee/', 'jh/']"
fi
export KNAPSACK_LOG_LEVEL="debug"
@@ -131,7 +135,7 @@ function rspec_paralellized_job() {
cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
- pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
+ pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
fi
diff --git a/scripts/rubocop-max-files-in-cache-check b/scripts/rubocop-max-files-in-cache-check
new file mode 100755
index 00000000000..5b422d0a0f4
--- /dev/null
+++ b/scripts/rubocop-max-files-in-cache-check
@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'yaml'
+
+MINIMUM_MAX_FILES_IN_CACHE_MARGIN = 1.05
+RECOMMENDED_MAX_FILES_IN_CACHE_MARGIN = 1.25
+RUBOCOP_LIST_TARGET_FILES_COMMAND = 'bundle exec rubocop --list-target-files | wc -l'
+
+RuboCopMaxFilesInCacheIsTooSmall = Class.new(StandardError)
+
+rubocop_target_files_count = `#{RUBOCOP_LIST_TARGET_FILES_COMMAND}`.strip.to_i
+
+raise Error, "#{RUBOCOP_LIST_TARGET_FILES_COMMAND} failed with status #{$?}!" if rubocop_target_files_count == 0
+
+rubocop_target_files_count = rubocop_target_files_count.to_i
+rubocop_current_max_files_in_cache = YAML.load_file(File.expand_path('../.rubocop.yml', __dir__)).dig('AllCops', 'MaxFilesInCache').to_i
+minimum_max_files_in_cache = (rubocop_target_files_count * MINIMUM_MAX_FILES_IN_CACHE_MARGIN).round(-3)
+
+# We want AllCops.MaxFilesInCache to be at least 5% above the actual files count at any time to give us enough time to increase it accordingly
+if rubocop_current_max_files_in_cache <= minimum_max_files_in_cache
+ recommended_max_files_in_cache = (rubocop_target_files_count * RECOMMENDED_MAX_FILES_IN_CACHE_MARGIN).round(-3)
+ raise RuboCopMaxFilesInCacheIsTooSmall, "Current count of RuboCop target file is #{rubocop_target_files_count} but AllCops.MaxFilesInCache is set to #{rubocop_current_max_files_in_cache}. We recommend to increase it to #{recommended_max_files_in_cache}."
+else
+ puts "Current count of RuboCop target file is #{rubocop_target_files_count} and AllCops.MaxFilesInCache is set to #{rubocop_current_max_files_in_cache}. All good."
+ exit(0)
+end
diff --git a/scripts/static-analysis b/scripts/static-analysis
index de5a1b407f9..f50e4a24b58 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -54,6 +54,7 @@ class StaticAnalysis
Task.new(%w[bin/rake gettext:lint], 85),
Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),
Task.new(%w[bin/rake lint:static_verification], 35),
+ Task.new(%w[scripts/rubocop-max-files-in-cache-check], 20),
Task.new(%w[bin/rake config_lint], 10),
Task.new(%w[bin/rake gitlab:sidekiq:all_queues_yml:check], 15),
(Gitlab.ee? ? Task.new(%w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check], 11) : nil),
@@ -65,7 +66,9 @@ class StaticAnalysis
].compact.freeze
def run_tasks!(options = {})
- node_assignment = tasks_to_run((ENV['CI_NODE_TOTAL'] || 1).to_i)[(ENV['CI_NODE_INDEX'] || 1).to_i - 1]
+ total_nodes = (ENV['CI_NODE_TOTAL'] || 1).to_i
+ current_node_number = (ENV['CI_NODE_INDEX'] || 1).to_i
+ node_assignment = tasks_to_run(total_nodes)[current_node_number - 1]
if options[:dry_run]
puts "Dry-run mode!"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index d2e8c151438..faabc151963 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -60,7 +60,7 @@ function setup_db_user_only() {
function setup_db() {
run_timed_command "setup_db_user_only"
- run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee"
+ run_timed_command_with_metric "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee" "setup_db"
}
function install_api_client_dependencies_with_apk() {
@@ -78,15 +78,27 @@ function install_tff_gem() {
function run_timed_command() {
local cmd="${1}"
+ local metric_name="${2}"
+ local timed_metric_file
local start=$(date +%s)
+
echosuccess "\$ ${cmd}"
eval "${cmd}"
+
local ret=$?
local end=$(date +%s)
local runtime=$((end-start))
if [[ $ret -eq 0 ]]; then
echosuccess "==> '${cmd}' succeeded in ${runtime} seconds."
+
+ if [[ -n "${metric_name}" ]]; then
+ timed_metric_file=$(timed_metric_file $metric_name)
+ echo "# TYPE ${metric_name} gauge" > "${timed_metric_file}"
+ echo "# UNIT ${metric_name} seconds" >> "${timed_metric_file}"
+ echo "${metric_name} ${runtime}" >> "${timed_metric_file}"
+ fi
+
return 0
else
echoerr "==> '${cmd}' failed (${ret}) in ${runtime} seconds."
@@ -94,6 +106,26 @@ function run_timed_command() {
fi
}
+function run_timed_command_with_metric() {
+ local cmd="${1}"
+ local metric_name="${2}"
+ local metrics_file=${METRICS_FILE:-metrics.txt}
+
+ run_timed_command "${cmd}" "${metric_name}"
+
+ local ret=$?
+
+ cat $(timed_metric_file $metric_name) >> "${metrics_file}"
+
+ return $ret
+}
+
+function timed_metric_file() {
+ local metric_name="${1}"
+
+ echo "$(pwd)/tmp/duration_${metric_name}.txt"
+}
+
function echoerr() {
local header="${2}"
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index ca73d4f5f7a..ee86f9ecde5 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -102,14 +102,14 @@ tests = [
{
explanation: 'Migration should map to its non-timestamped spec',
- source: 'db/migrate/20191023152913_add_default_and_free_plans.rb',
- expected: ['spec/migrations/add_default_and_free_plans_spec.rb']
+ source: 'db/migrate/20200116175538_update_timestamp_softwarelicensespolicy.rb',
+ expected: ['spec/migrations/update_timestamp_softwarelicensespolicy_spec.rb']
},
{
explanation: 'Migration should map to its timestamped spec',
- source: 'db/post_migrate/20190924152703_migrate_issue_trackers_data.rb',
- expected: ['spec/migrations/20190924152703_migrate_issue_trackers_data_spec.rb']
+ source: 'db/post_migrate/20210915022415_cleanup_bigint_conversion_for_ci_builds.rb',
+ expected: ['spec/migrations/20210915022415_cleanup_bigint_conversion_for_ci_builds_spec.rb']
},
{