diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gitaly-test-build | 2 | ||||
| -rwxr-xr-x | scripts/gitaly-test-spawn | 1 | ||||
| -rw-r--r-- | scripts/gitaly_test.rb | 30 | ||||
| -rwxr-xr-x | scripts/lint-rugged | 1 | ||||
| -rwxr-xr-x | scripts/perf/query_limiting_report.rb | 167 | ||||
| -rw-r--r-- | scripts/review_apps/base-config.yaml | 12 | ||||
| -rw-r--r-- | scripts/rspec_helpers.sh | 6 | ||||
| -rwxr-xr-x | scripts/static-analysis | 2 | ||||
| -rwxr-xr-x | scripts/trigger-build | 86 | ||||
| -rw-r--r-- | scripts/utils.sh | 6 | ||||
| -rwxr-xr-x | scripts/verify-tff-mapping | 2 |
11 files changed, 243 insertions, 72 deletions
diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build index bb561e2906a..849c08df527 100755 --- a/scripts/gitaly-test-build +++ b/scripts/gitaly-test-build @@ -13,6 +13,8 @@ class GitalyTestBuild include GitalyTest def run + set_bundler_config + abort 'gitaly build failed' unless build_gitaly ensure_gitlab_shell_secret! diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn index 8547d0b13e4..7cb9ea803f8 100755 --- a/scripts/gitaly-test-spawn +++ b/scripts/gitaly-test-spawn @@ -9,6 +9,7 @@ class GitalyTestSpawn include GitalyTest def run + set_bundler_config install_gitaly_gems if ENV['CI'] check_gitaly_config! diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb index 2262870eb96..b7ea9cd628e 100644 --- a/scripts/gitaly_test.rb +++ b/scripts/gitaly_test.rb @@ -12,11 +12,11 @@ require 'logger' module GitalyTest LOGGER = begin - default_name = ENV['CI'] ? 'DEBUG' : 'WARN' - level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase - level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse - Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg }) - end + default_name = ENV['CI'] ? 'DEBUG' : 'WARN' + level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase + level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse + Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg }) + end def tmp_tests_gitaly_dir File.expand_path('../tmp/tests/gitaly', __dir__) @@ -34,16 +34,19 @@ module GitalyTest File.join(tmp_tests_gitaly_dir, 'ruby', 'Gemfile') end + def gemfile_dir + File.dirname(gemfile) + end + def gitlab_shell_secret_file File.join(tmp_tests_gitlab_shell_dir, '.gitlab_shell_secret') end def env - env_hash = { + { 'HOME' => File.expand_path('tmp/tests'), 'GEM_PATH' => Gem.path.join(':'), - 'BUNDLE_APP_CONFIG' => File.join(File.dirname(gemfile), '.bundle/config'), - 'BUNDLE_FLAGS' => "--jobs=4 --retry=3", + 'BUNDLE_APP_CONFIG' => File.join(gemfile_dir, '.bundle'), 'BUNDLE_INSTALL_FLAGS' => nil, 'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil, @@ -51,14 +54,19 @@ module GitalyTest # Git hooks can't run during tests as the internal API is not running. 'GITALY_TESTING_NO_GIT_HOOKS' => "1" } + end + + # rubocop:disable GitlabSecurity/SystemCommandInjection + def set_bundler_config + system('bundle config set --local jobs 4', chdir: gemfile_dir) + system('bundle config set --local retry 3', chdir: gemfile_dir) if ENV['CI'] bundle_path = File.expand_path('../vendor/gitaly-ruby', __dir__) - env_hash['BUNDLE_FLAGS'] += " --path=#{bundle_path}" + system('bundle', 'config', 'set', '--local', 'path', bundle_path, chdir: gemfile_dir) end - - env_hash end + # rubocop:enable GitlabSecurity/SystemCommandInjection def config_path(service) case service diff --git a/scripts/lint-rugged b/scripts/lint-rugged index 038fd5199c2..ae5eddbe3b4 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -29,6 +29,7 @@ ALLOWED = [ 'config/initializers/lograge.rb', 'lib/gitlab/grape_logging/loggers/perf_logger.rb', 'lib/gitlab/instrumentation_helper.rb', + 'lib/gitlab/sidekiq_middleware/instrumentation_logger.rb', 'lib/gitlab/rugged_instrumentation.rb', 'lib/peek/views/rugged.rb' ].freeze diff --git a/scripts/perf/query_limiting_report.rb b/scripts/perf/query_limiting_report.rb new file mode 100755 index 00000000000..2f263eeb567 --- /dev/null +++ b/scripts/perf/query_limiting_report.rb @@ -0,0 +1,167 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +#### +# Prints a report which helps reconcile occurrences of the `QueryLimiting.disable(ISSUE_LINK)` +# allowlist block against the corresponding open issues. +# +# If everything is consistent, the script should ideally not report any issues or code lines, +# other than possibly remaining "calls with no issue iid" which use variables/etc. +# +# - See https://gitlab.com/gitlab-org/gitlab/-/issues/325640 +# - See https://gitlab.com/groups/gitlab-org/-/epics/5670 + +require 'rubygems' +require 'gitlab' +require 'optparse' + +class QueryLimitingReport + GITLAB_PROJECT_ID = 278964 # gitlab-org/gitlab project + ISSUES_SEARCH_LABEL = 'querylimiting-disable' + CODE_LINES_SEARCH_STRING = 'QueryLimiting.disable' + PAGINATION_LIMIT = 500 + + DEFAULT_OPTIONS = { + api_token: ENV['API_TOKEN'] + }.freeze + + def initialize(options) + @options = options + + Gitlab.configure do |config| + config.endpoint = 'https://gitlab.com/api/v4' + config.private_token = options.fetch(:api_token) + end + end + + def execute + # PLAN: + # Read all issues matching criteria and extract array of issue iids + # Find all code references and extract issue iids + # Print list of all issues without code references + # Print list of all code references issue iids that don't have search label + # Print list of all code references with no issue iids (i.e. dynamic or variable argument) + + total_issues = find_issues_by_label(ISSUES_SEARCH_LABEL) + issues = total_issues.select { |issue| issue[:state] == 'opened' } + code_lines = find_code_lines + + code_lines_grouped = code_lines.group_by { |code_line| code_line[:has_issue_iid] } + code_lines_without_issue_iid = code_lines_grouped[false] + code_lines_with_issue_iid = code_lines_grouped[true] + + all_issue_iids_in_code_lines = code_lines_with_issue_iid.map { |line| line[:issue_iid] } + + issues_without_code_references = issues.reject do |issue| + all_issue_iids_in_code_lines.include?(issue[:iid]) + end + + all_issue_iids = issues.map { |issue| issue[:iid] } + code_lines_with_missing_issues = code_lines_with_issue_iid.reject do |code_line| + all_issue_iids.include?(code_line[:issue_iid]) + end + + puts "\n\n\nREPORT:" + + puts "\n\nFound #{total_issues.length} total issues with '#{ISSUES_SEARCH_LABEL}' search label, #{issues.length} are still opened..." + puts "\n\nFound #{code_lines.length} total occurrences of '#{CODE_LINES_SEARCH_STRING}' in code..." + + puts "\n" + '-' * 80 + + puts "\n\nIssues without any '#{CODE_LINES_SEARCH_STRING}' code references (#{issues_without_code_references.length} total):" + pp issues_without_code_references + + puts "\n" + '-' * 80 + + puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with references to an issue which doesn't have '#{ISSUES_SEARCH_LABEL}' search label (#{code_lines_with_missing_issues.length} total):" + pp code_lines_with_missing_issues + + puts "\n" + '-' * 80 + + puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with no issue iid (#{code_lines_without_issue_iid&.length || 0} total):" + pp code_lines_without_issue_iid + end + + private + + attr_reader :options + + def find_issues_by_label(label) + issues = [] + + puts("Finding issues by label #{label}...") + paginated_issues = Gitlab.issues(GITLAB_PROJECT_ID, 'labels' => label) + paginated_issues.paginate_with_limit(PAGINATION_LIMIT) do |item| + item_hash = item.to_hash + + issue_iid = item_hash.fetch('iid') + issue = { + iid: issue_iid, + state: item_hash.fetch('state'), + title: item_hash.fetch('title'), + issue_url: "https://gitlab.com/gitlab-org/gitlab/issues/#{issue_iid}" + } + + issues << issue + end + + issues + end + + def find_code_lines + code_lines = [] + + puts("Finding code lines...") + paginated_blobs = Gitlab.search_in_project(GITLAB_PROJECT_ID, 'blobs', CODE_LINES_SEARCH_STRING) + paginated_blobs.paginate_with_limit(PAGINATION_LIMIT) do |item| + item_hash = item.to_hash + + filename = item_hash.fetch('filename') + next if filename !~ /\.rb\Z/ + + file_contents = Gitlab.file_contents(GITLAB_PROJECT_ID, filename) + file_lines = file_contents.split("\n") + + file_lines.each_index do |index| + line = file_lines[index] + if line =~ /#{CODE_LINES_SEARCH_STRING}/ + issue_iid = line.slice(%r{issues/(\d+)\D}, 1) + line_number = index + 1 + code_line = { + file_location: "#{filename}:#{line_number}", + filename: filename, + line_number: line_number, + line: line, + issue_iid: issue_iid.to_i, + has_issue_iid: !issue_iid.nil? + } + code_lines << code_line + end + end + end + + code_lines.sort_by! { |line| "#{line[:filename]}-#{line[:line_number].to_s.rjust(4, '0')}" } + code_lines.map do |line| + line.delete(:filename) + line.delete(:line_number) + line + end + end +end + +if $0 == __FILE__ + options = QueryLimitingReport::DEFAULT_OPTIONS.dup + + OptionParser.new do |opts| + opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope. Can be set as an env variable 'API_TOKEN'.") do |value| + options[:api_token] = value + end + + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end + end.parse! + + QueryLimitingReport.new(options).execute +end diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml index cf55fca7452..7daf3f80efc 100644 --- a/scripts/review_apps/base-config.yaml +++ b/scripts/review_apps/base-config.yaml @@ -70,10 +70,10 @@ gitlab: resources: requests: cpu: 746m - memory: 1873M + memory: 2809M limits: cpu: 1119m - memory: 2809M + memory: 4214M deployment: readinessProbe: initialDelaySeconds: 5 # Default is 0 @@ -83,10 +83,10 @@ gitlab: resources: requests: cpu: 400m - memory: 50M + memory: 75M limits: cpu: 600m - memory: 75M + memory: 113M readinessProbe: initialDelaySeconds: 5 # Default is 0 periodSeconds: 15 # Default is 10 @@ -137,10 +137,10 @@ postgresql: enabled: false resources: requests: - cpu: 550m + cpu: 600m memory: 1000M limits: - cpu: 825m + cpu: 1300m memory: 1500M prometheus: install: false diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index d9957fb6ced..e23231e429b 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -97,6 +97,10 @@ function rspec_paralellized_job() { spec_folder_prefix="ee/" fi + if [[ "${test_tool}" =~ "-jh" ]]; then + spec_folder_prefix="jh/" + fi + export KNAPSACK_LOG_LEVEL="debug" export KNAPSACK_REPORT_PATH="knapsack/${report_name}_report.json" @@ -109,7 +113,7 @@ function rspec_paralellized_job() { cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}" if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then - pattern=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})") + pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})") export KNAPSACK_TEST_FILE_PATTERN="${pattern}" fi diff --git a/scripts/static-analysis b/scripts/static-analysis index 2442455e630..136b2966244 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -33,7 +33,7 @@ class StaticAnalysis %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 13, (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 13, %w[bin/rake config_lint] => 11, - %w[yarn run lint:stylelint] => 9, + %w[yarn run internal:stylelint] => 9, %w[scripts/lint-conflicts.sh] => 0.59, %w[yarn run block-dependencies] => 0.35, %w[scripts/lint-rugged] => 0.23, diff --git a/scripts/trigger-build b/scripts/trigger-build index 29d53609026..3a9301bda3b 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build @@ -136,13 +136,15 @@ module Trigger def extra_variables # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results - # and fallback to CI_COMMIT_SHA for the `detached` pipelines. - # We also set IMAGE_TAG so the GitLab and QA docker images are tagged with - # that SHA. + # and fallback to CI_COMMIT_SHA (merged result commit) for the non-MR pipelines. + # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results. + # We also set IMAGE_TAG so the GitLab Docker image is tagged with that SHA. source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'] { 'GITLAB_VERSION' => source_sha, 'IMAGE_TAG' => source_sha, + 'QA_IMAGE' => "#{ENV['CI_REGISTRY']}/#{ENV['CI_PROJECT_PATH']}/gitlab-ee-qa:#{ENV['CI_COMMIT_REF_SLUG']}", + 'SKIP_QA_DOCKER' => 'true', 'ALTERNATIVE_SOURCES' => 'true', 'SECURITY_SOURCES' => Trigger.security? ? 'true' : 'false', 'ee' => Trigger.ee? ? 'true' : 'false', @@ -214,15 +216,7 @@ module Trigger => If something doesn't work, drop a line in the #docs chat channel. MSG - # Create a remote branch in gitlab-docs and immediately cancel the pipeline - # to avoid race conditions, since a triggered pipeline will also run right - # after the branch creation. This only happens the very first time a branch - # is created and will be skipped in subsequent runs. Read more in - # https://gitlab.com/gitlab-org/gitlab-docs/issues/154. - # def deploy! - create_remote_branch! - cancel_latest_pipeline! invoke!.wait! display_success_message end @@ -231,31 +225,52 @@ module Trigger # Remove a remote branch in gitlab-docs. # def cleanup! - gitlab_client(:downstream).delete_branch(downstream_project_path, ref) - puts "=> Remote branch '#{downstream_project_path}' deleted" + environment = gitlab_client(:downstream).environments(downstream_project_path, name: downstream_environment).first + return unless environment + + environment = gitlab_client(:downstream).stop_environment(downstream_project_path, environment.id) + if environment.state == 'stopped' + puts "=> Downstream environment '#{downstream_environment}' stopped" + else + puts "=> Downstream environment '#{downstream_environment}' failed to stop." + end end private + def downstream_environment + "review/#{ref}#{review_slug}" + end + + # We prepend the `-` here because we cannot use variable substitution in `environment.name`/`environment.url` + # Some projects (e.g. `omnibus-gitlab`) use this script for branch pipelines, so we fallback to using `CI_COMMIT_REF_SLUG` for those cases. + def review_slug + identifier = ENV['CI_MERGE_REQUEST_IID'] || ENV['CI_COMMIT_REF_SLUG'] + + "-#{project_slug}-#{identifier}" + end + def downstream_project_path ENV['DOCS_PROJECT_PATH'] || 'gitlab-org/gitlab-docs' end def ref - if ENV['CI_MERGE_REQUEST_IID'].nil? - "docs-preview-#{slug}-#{ENV['CI_COMMIT_REF_SLUG']}" - else - "docs-preview-#{slug}-#{ENV['CI_MERGE_REQUEST_IID']}" - end + ENV['DOCS_BRANCH'] || 'master' + end + + # `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job" + def trigger_token + ENV['DOCS_TRIGGER_TOKEN'] end def extra_variables { - "BRANCH_#{slug.upcase}" => ENV['CI_COMMIT_REF_NAME'] + "BRANCH_#{project_slug.upcase}" => ENV['CI_COMMIT_REF_NAME'], + "REVIEW_SLUG" => review_slug } end - def slug + def project_slug case ENV['CI_PROJECT_PATH'] when 'gitlab-org/gitlab-foss' 'ce' @@ -270,37 +285,14 @@ module Trigger end end + # app_url is the URL of the `gitlab-docs` Review App URL defined in + # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/b38038132cf82a24271bbb294dead7c2f529e275/.gitlab-ci.yml#L383 def app_url - "http://#{ref}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{slug}" - end - - def create_remote_branch! - gitlab_client(:downstream).create_branch(downstream_project_path, ref, 'master') - puts "=> Remote branch '#{ref}' created" - rescue Gitlab::Error::BadRequest - puts "=> Remote branch '#{ref}' already exists!" - end - - def cancel_latest_pipeline! - pipelines = nil - - # Wait until the pipeline is started - loop do - sleep 1 - puts "=> Waiting for pipeline to start..." - pipelines = gitlab_client(:downstream).pipelines(downstream_project_path, { ref: ref }) - break if pipelines.any? - end - - # Get the first pipeline ID which should be the only one for the branch - pipeline_id = pipelines.first.id - - # Cancel the pipeline - gitlab_client(:downstream).cancel_pipeline(downstream_project_path, pipeline_id) + "http://#{ref}#{review_slug}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{project_slug}" end def display_success_message - format(SUCCESS_MESSAGE, app_url: app_url) + puts format(SUCCESS_MESSAGE, app_url: app_url) end end diff --git a/scripts/utils.sh b/scripts/utils.sh index c598afc4582..2e9839e4df8 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -26,13 +26,9 @@ function install_api_client_dependencies_with_apk() { apk add --update openssl curl jq } -function install_api_client_dependencies_with_apt() { - apt update && apt install jq -y -} - function install_gitlab_gem() { gem install httparty --no-document --version 0.18.1 - gem install gitlab --no-document --version 4.14.1 + gem install gitlab --no-document --version 4.17.0 } function install_tff_gem() { diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping index 5bc4b23f99f..e18a14e17d6 100755 --- a/scripts/verify-tff-mapping +++ b/scripts/verify-tff-mapping @@ -8,7 +8,7 @@ require 'set' # The verification depend on the presence of actual test files, # so they would fail if one of the test files mentioned here is deleted. # To minimize the chance of this test failing due to unrelated changes, -# the test files are chosen to be critical files that are unlikely to be deleted in a typical Merge Request +# the test files are chosen to be critical files that are unlikely to be deleted in a typical merge request tests = [ { explanation: 'EE code should map to respective spec', |