summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-04-20 23:50:22 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2021-04-20 23:50:22 +0000
commit9dc93a4519d9d5d7be48ff274127136236a3adb3 (patch)
tree70467ae3692a0e35e5ea56bcb803eb512a10bedb /scripts
parent4b0f34b6d759d6299322b3a54453e930c6121ff0 (diff)
downloadgitlab-ce-9dc93a4519d9d5d7be48ff274127136236a3adb3.tar.gz
Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/gitaly-test-build2
-rwxr-xr-xscripts/gitaly-test-spawn1
-rw-r--r--scripts/gitaly_test.rb30
-rwxr-xr-xscripts/lint-rugged1
-rwxr-xr-xscripts/perf/query_limiting_report.rb167
-rw-r--r--scripts/review_apps/base-config.yaml12
-rw-r--r--scripts/rspec_helpers.sh6
-rwxr-xr-xscripts/static-analysis2
-rwxr-xr-xscripts/trigger-build86
-rw-r--r--scripts/utils.sh6
-rwxr-xr-xscripts/verify-tff-mapping2
11 files changed, 243 insertions, 72 deletions
diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build
index bb561e2906a..849c08df527 100755
--- a/scripts/gitaly-test-build
+++ b/scripts/gitaly-test-build
@@ -13,6 +13,8 @@ class GitalyTestBuild
include GitalyTest
def run
+ set_bundler_config
+
abort 'gitaly build failed' unless build_gitaly
ensure_gitlab_shell_secret!
diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn
index 8547d0b13e4..7cb9ea803f8 100755
--- a/scripts/gitaly-test-spawn
+++ b/scripts/gitaly-test-spawn
@@ -9,6 +9,7 @@ class GitalyTestSpawn
include GitalyTest
def run
+ set_bundler_config
install_gitaly_gems if ENV['CI']
check_gitaly_config!
diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb
index 2262870eb96..b7ea9cd628e 100644
--- a/scripts/gitaly_test.rb
+++ b/scripts/gitaly_test.rb
@@ -12,11 +12,11 @@ require 'logger'
module GitalyTest
LOGGER = begin
- default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
- level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
- level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse
- Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg })
- end
+ default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
+ level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
+ level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse
+ Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg })
+ end
def tmp_tests_gitaly_dir
File.expand_path('../tmp/tests/gitaly', __dir__)
@@ -34,16 +34,19 @@ module GitalyTest
File.join(tmp_tests_gitaly_dir, 'ruby', 'Gemfile')
end
+ def gemfile_dir
+ File.dirname(gemfile)
+ end
+
def gitlab_shell_secret_file
File.join(tmp_tests_gitlab_shell_dir, '.gitlab_shell_secret')
end
def env
- env_hash = {
+ {
'HOME' => File.expand_path('tmp/tests'),
'GEM_PATH' => Gem.path.join(':'),
- 'BUNDLE_APP_CONFIG' => File.join(File.dirname(gemfile), '.bundle/config'),
- 'BUNDLE_FLAGS' => "--jobs=4 --retry=3",
+ 'BUNDLE_APP_CONFIG' => File.join(gemfile_dir, '.bundle'),
'BUNDLE_INSTALL_FLAGS' => nil,
'BUNDLE_GEMFILE' => gemfile,
'RUBYOPT' => nil,
@@ -51,14 +54,19 @@ module GitalyTest
# Git hooks can't run during tests as the internal API is not running.
'GITALY_TESTING_NO_GIT_HOOKS' => "1"
}
+ end
+
+ # rubocop:disable GitlabSecurity/SystemCommandInjection
+ def set_bundler_config
+ system('bundle config set --local jobs 4', chdir: gemfile_dir)
+ system('bundle config set --local retry 3', chdir: gemfile_dir)
if ENV['CI']
bundle_path = File.expand_path('../vendor/gitaly-ruby', __dir__)
- env_hash['BUNDLE_FLAGS'] += " --path=#{bundle_path}"
+ system('bundle', 'config', 'set', '--local', 'path', bundle_path, chdir: gemfile_dir)
end
-
- env_hash
end
+ # rubocop:enable GitlabSecurity/SystemCommandInjection
def config_path(service)
case service
diff --git a/scripts/lint-rugged b/scripts/lint-rugged
index 038fd5199c2..ae5eddbe3b4 100755
--- a/scripts/lint-rugged
+++ b/scripts/lint-rugged
@@ -29,6 +29,7 @@ ALLOWED = [
'config/initializers/lograge.rb',
'lib/gitlab/grape_logging/loggers/perf_logger.rb',
'lib/gitlab/instrumentation_helper.rb',
+ 'lib/gitlab/sidekiq_middleware/instrumentation_logger.rb',
'lib/gitlab/rugged_instrumentation.rb',
'lib/peek/views/rugged.rb'
].freeze
diff --git a/scripts/perf/query_limiting_report.rb b/scripts/perf/query_limiting_report.rb
new file mode 100755
index 00000000000..2f263eeb567
--- /dev/null
+++ b/scripts/perf/query_limiting_report.rb
@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+####
+# Prints a report which helps reconcile occurrences of the `QueryLimiting.disable(ISSUE_LINK)`
+# allowlist block against the corresponding open issues.
+#
+# If everything is consistent, the script should ideally not report any issues or code lines,
+# other than possibly remaining "calls with no issue iid" which use variables/etc.
+#
+# - See https://gitlab.com/gitlab-org/gitlab/-/issues/325640
+# - See https://gitlab.com/groups/gitlab-org/-/epics/5670
+
+require 'rubygems'
+require 'gitlab'
+require 'optparse'
+
+class QueryLimitingReport
+ GITLAB_PROJECT_ID = 278964 # gitlab-org/gitlab project
+ ISSUES_SEARCH_LABEL = 'querylimiting-disable'
+ CODE_LINES_SEARCH_STRING = 'QueryLimiting.disable'
+ PAGINATION_LIMIT = 500
+
+ DEFAULT_OPTIONS = {
+ api_token: ENV['API_TOKEN']
+ }.freeze
+
+ def initialize(options)
+ @options = options
+
+ Gitlab.configure do |config|
+ config.endpoint = 'https://gitlab.com/api/v4'
+ config.private_token = options.fetch(:api_token)
+ end
+ end
+
+ def execute
+ # PLAN:
+ # Read all issues matching criteria and extract array of issue iids
+ # Find all code references and extract issue iids
+ # Print list of all issues without code references
+ # Print list of all code references issue iids that don't have search label
+ # Print list of all code references with no issue iids (i.e. dynamic or variable argument)
+
+ total_issues = find_issues_by_label(ISSUES_SEARCH_LABEL)
+ issues = total_issues.select { |issue| issue[:state] == 'opened' }
+ code_lines = find_code_lines
+
+ code_lines_grouped = code_lines.group_by { |code_line| code_line[:has_issue_iid] }
+ code_lines_without_issue_iid = code_lines_grouped[false]
+ code_lines_with_issue_iid = code_lines_grouped[true]
+
+ all_issue_iids_in_code_lines = code_lines_with_issue_iid.map { |line| line[:issue_iid] }
+
+ issues_without_code_references = issues.reject do |issue|
+ all_issue_iids_in_code_lines.include?(issue[:iid])
+ end
+
+ all_issue_iids = issues.map { |issue| issue[:iid] }
+ code_lines_with_missing_issues = code_lines_with_issue_iid.reject do |code_line|
+ all_issue_iids.include?(code_line[:issue_iid])
+ end
+
+ puts "\n\n\nREPORT:"
+
+ puts "\n\nFound #{total_issues.length} total issues with '#{ISSUES_SEARCH_LABEL}' search label, #{issues.length} are still opened..."
+ puts "\n\nFound #{code_lines.length} total occurrences of '#{CODE_LINES_SEARCH_STRING}' in code..."
+
+ puts "\n" + '-' * 80
+
+ puts "\n\nIssues without any '#{CODE_LINES_SEARCH_STRING}' code references (#{issues_without_code_references.length} total):"
+ pp issues_without_code_references
+
+ puts "\n" + '-' * 80
+
+ puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with references to an issue which doesn't have '#{ISSUES_SEARCH_LABEL}' search label (#{code_lines_with_missing_issues.length} total):"
+ pp code_lines_with_missing_issues
+
+ puts "\n" + '-' * 80
+
+ puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with no issue iid (#{code_lines_without_issue_iid&.length || 0} total):"
+ pp code_lines_without_issue_iid
+ end
+
+ private
+
+ attr_reader :options
+
+ def find_issues_by_label(label)
+ issues = []
+
+ puts("Finding issues by label #{label}...")
+ paginated_issues = Gitlab.issues(GITLAB_PROJECT_ID, 'labels' => label)
+ paginated_issues.paginate_with_limit(PAGINATION_LIMIT) do |item|
+ item_hash = item.to_hash
+
+ issue_iid = item_hash.fetch('iid')
+ issue = {
+ iid: issue_iid,
+ state: item_hash.fetch('state'),
+ title: item_hash.fetch('title'),
+ issue_url: "https://gitlab.com/gitlab-org/gitlab/issues/#{issue_iid}"
+ }
+
+ issues << issue
+ end
+
+ issues
+ end
+
+ def find_code_lines
+ code_lines = []
+
+ puts("Finding code lines...")
+ paginated_blobs = Gitlab.search_in_project(GITLAB_PROJECT_ID, 'blobs', CODE_LINES_SEARCH_STRING)
+ paginated_blobs.paginate_with_limit(PAGINATION_LIMIT) do |item|
+ item_hash = item.to_hash
+
+ filename = item_hash.fetch('filename')
+ next if filename !~ /\.rb\Z/
+
+ file_contents = Gitlab.file_contents(GITLAB_PROJECT_ID, filename)
+ file_lines = file_contents.split("\n")
+
+ file_lines.each_index do |index|
+ line = file_lines[index]
+ if line =~ /#{CODE_LINES_SEARCH_STRING}/
+ issue_iid = line.slice(%r{issues/(\d+)\D}, 1)
+ line_number = index + 1
+ code_line = {
+ file_location: "#{filename}:#{line_number}",
+ filename: filename,
+ line_number: line_number,
+ line: line,
+ issue_iid: issue_iid.to_i,
+ has_issue_iid: !issue_iid.nil?
+ }
+ code_lines << code_line
+ end
+ end
+ end
+
+ code_lines.sort_by! { |line| "#{line[:filename]}-#{line[:line_number].to_s.rjust(4, '0')}" }
+ code_lines.map do |line|
+ line.delete(:filename)
+ line.delete(:line_number)
+ line
+ end
+ end
+end
+
+if $0 == __FILE__
+ options = QueryLimitingReport::DEFAULT_OPTIONS.dup
+
+ OptionParser.new do |opts|
+ opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope. Can be set as an env variable 'API_TOKEN'.") do |value|
+ options[:api_token] = value
+ end
+
+ opts.on("-h", "--help", "Prints this help") do
+ puts opts
+ exit
+ end
+ end.parse!
+
+ QueryLimitingReport.new(options).execute
+end
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index cf55fca7452..7daf3f80efc 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -70,10 +70,10 @@ gitlab:
resources:
requests:
cpu: 746m
- memory: 1873M
+ memory: 2809M
limits:
cpu: 1119m
- memory: 2809M
+ memory: 4214M
deployment:
readinessProbe:
initialDelaySeconds: 5 # Default is 0
@@ -83,10 +83,10 @@ gitlab:
resources:
requests:
cpu: 400m
- memory: 50M
+ memory: 75M
limits:
cpu: 600m
- memory: 75M
+ memory: 113M
readinessProbe:
initialDelaySeconds: 5 # Default is 0
periodSeconds: 15 # Default is 10
@@ -137,10 +137,10 @@ postgresql:
enabled: false
resources:
requests:
- cpu: 550m
+ cpu: 600m
memory: 1000M
limits:
- cpu: 825m
+ cpu: 1300m
memory: 1500M
prometheus:
install: false
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index d9957fb6ced..e23231e429b 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -97,6 +97,10 @@ function rspec_paralellized_job() {
spec_folder_prefix="ee/"
fi
+ if [[ "${test_tool}" =~ "-jh" ]]; then
+ spec_folder_prefix="jh/"
+ fi
+
export KNAPSACK_LOG_LEVEL="debug"
export KNAPSACK_REPORT_PATH="knapsack/${report_name}_report.json"
@@ -109,7 +113,7 @@ function rspec_paralellized_job() {
cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
- pattern=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
+ pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
fi
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 2442455e630..136b2966244 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -33,7 +33,7 @@ class StaticAnalysis
%w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 13,
(Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 13,
%w[bin/rake config_lint] => 11,
- %w[yarn run lint:stylelint] => 9,
+ %w[yarn run internal:stylelint] => 9,
%w[scripts/lint-conflicts.sh] => 0.59,
%w[yarn run block-dependencies] => 0.35,
%w[scripts/lint-rugged] => 0.23,
diff --git a/scripts/trigger-build b/scripts/trigger-build
index 29d53609026..3a9301bda3b 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -136,13 +136,15 @@ module Trigger
def extra_variables
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
- # and fallback to CI_COMMIT_SHA for the `detached` pipelines.
- # We also set IMAGE_TAG so the GitLab and QA docker images are tagged with
- # that SHA.
+ # and fallback to CI_COMMIT_SHA (merged result commit) for the non-MR pipelines.
+ # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results.
+ # We also set IMAGE_TAG so the GitLab Docker image is tagged with that SHA.
source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
{
'GITLAB_VERSION' => source_sha,
'IMAGE_TAG' => source_sha,
+ 'QA_IMAGE' => "#{ENV['CI_REGISTRY']}/#{ENV['CI_PROJECT_PATH']}/gitlab-ee-qa:#{ENV['CI_COMMIT_REF_SLUG']}",
+ 'SKIP_QA_DOCKER' => 'true',
'ALTERNATIVE_SOURCES' => 'true',
'SECURITY_SOURCES' => Trigger.security? ? 'true' : 'false',
'ee' => Trigger.ee? ? 'true' : 'false',
@@ -214,15 +216,7 @@ module Trigger
=> If something doesn't work, drop a line in the #docs chat channel.
MSG
- # Create a remote branch in gitlab-docs and immediately cancel the pipeline
- # to avoid race conditions, since a triggered pipeline will also run right
- # after the branch creation. This only happens the very first time a branch
- # is created and will be skipped in subsequent runs. Read more in
- # https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
- #
def deploy!
- create_remote_branch!
- cancel_latest_pipeline!
invoke!.wait!
display_success_message
end
@@ -231,31 +225,52 @@ module Trigger
# Remove a remote branch in gitlab-docs.
#
def cleanup!
- gitlab_client(:downstream).delete_branch(downstream_project_path, ref)
- puts "=> Remote branch '#{downstream_project_path}' deleted"
+ environment = gitlab_client(:downstream).environments(downstream_project_path, name: downstream_environment).first
+ return unless environment
+
+ environment = gitlab_client(:downstream).stop_environment(downstream_project_path, environment.id)
+ if environment.state == 'stopped'
+ puts "=> Downstream environment '#{downstream_environment}' stopped"
+ else
+ puts "=> Downstream environment '#{downstream_environment}' failed to stop."
+ end
end
private
+ def downstream_environment
+ "review/#{ref}#{review_slug}"
+ end
+
+ # We prepend the `-` here because we cannot use variable substitution in `environment.name`/`environment.url`
+ # Some projects (e.g. `omnibus-gitlab`) use this script for branch pipelines, so we fallback to using `CI_COMMIT_REF_SLUG` for those cases.
+ def review_slug
+ identifier = ENV['CI_MERGE_REQUEST_IID'] || ENV['CI_COMMIT_REF_SLUG']
+
+ "-#{project_slug}-#{identifier}"
+ end
+
def downstream_project_path
ENV['DOCS_PROJECT_PATH'] || 'gitlab-org/gitlab-docs'
end
def ref
- if ENV['CI_MERGE_REQUEST_IID'].nil?
- "docs-preview-#{slug}-#{ENV['CI_COMMIT_REF_SLUG']}"
- else
- "docs-preview-#{slug}-#{ENV['CI_MERGE_REQUEST_IID']}"
- end
+ ENV['DOCS_BRANCH'] || 'master'
+ end
+
+ # `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job"
+ def trigger_token
+ ENV['DOCS_TRIGGER_TOKEN']
end
def extra_variables
{
- "BRANCH_#{slug.upcase}" => ENV['CI_COMMIT_REF_NAME']
+ "BRANCH_#{project_slug.upcase}" => ENV['CI_COMMIT_REF_NAME'],
+ "REVIEW_SLUG" => review_slug
}
end
- def slug
+ def project_slug
case ENV['CI_PROJECT_PATH']
when 'gitlab-org/gitlab-foss'
'ce'
@@ -270,37 +285,14 @@ module Trigger
end
end
+ # app_url is the URL of the `gitlab-docs` Review App URL defined in
+ # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/b38038132cf82a24271bbb294dead7c2f529e275/.gitlab-ci.yml#L383
def app_url
- "http://#{ref}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{slug}"
- end
-
- def create_remote_branch!
- gitlab_client(:downstream).create_branch(downstream_project_path, ref, 'master')
- puts "=> Remote branch '#{ref}' created"
- rescue Gitlab::Error::BadRequest
- puts "=> Remote branch '#{ref}' already exists!"
- end
-
- def cancel_latest_pipeline!
- pipelines = nil
-
- # Wait until the pipeline is started
- loop do
- sleep 1
- puts "=> Waiting for pipeline to start..."
- pipelines = gitlab_client(:downstream).pipelines(downstream_project_path, { ref: ref })
- break if pipelines.any?
- end
-
- # Get the first pipeline ID which should be the only one for the branch
- pipeline_id = pipelines.first.id
-
- # Cancel the pipeline
- gitlab_client(:downstream).cancel_pipeline(downstream_project_path, pipeline_id)
+ "http://#{ref}#{review_slug}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{project_slug}"
end
def display_success_message
- format(SUCCESS_MESSAGE, app_url: app_url)
+ puts format(SUCCESS_MESSAGE, app_url: app_url)
end
end
diff --git a/scripts/utils.sh b/scripts/utils.sh
index c598afc4582..2e9839e4df8 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -26,13 +26,9 @@ function install_api_client_dependencies_with_apk() {
apk add --update openssl curl jq
}
-function install_api_client_dependencies_with_apt() {
- apt update && apt install jq -y
-}
-
function install_gitlab_gem() {
gem install httparty --no-document --version 0.18.1
- gem install gitlab --no-document --version 4.14.1
+ gem install gitlab --no-document --version 4.17.0
}
function install_tff_gem() {
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index 5bc4b23f99f..e18a14e17d6 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -8,7 +8,7 @@ require 'set'
# The verification depend on the presence of actual test files,
# so they would fail if one of the test files mentioned here is deleted.
# To minimize the chance of this test failing due to unrelated changes,
-# the test files are chosen to be critical files that are unlikely to be deleted in a typical Merge Request
+# the test files are chosen to be critical files that are unlikely to be deleted in a typical merge request
tests = [
{
explanation: 'EE code should map to respective spec',