diff options
Diffstat (limited to 'lib/gitlab')
42 files changed, 456 insertions, 84 deletions
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8e5a985edd7..7de66539848 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -14,6 +14,25 @@ module Gitlab DEFAULT_SCOPES = [:api].freeze class << self + def omniauth_customized_providers + @omniauth_customized_providers ||= %w[bitbucket jwt] + end + + def omniauth_setup_providers(provider_names) + provider_names.each do |provider| + omniauth_setup_a_provider(provider) + end + end + + def omniauth_setup_a_provider(provider) + case provider + when 'kerberos' + require 'omniauth-kerberos' + when *omniauth_customized_providers + require_dependency "omni_auth/strategies/#{provider}" + end + end + def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -221,7 +240,7 @@ module Gitlab return unless login == 'gitlab-ci-token' return unless password - build = ::Ci::Build.running.find_by_token(password) + build = find_build_by_token(password) return unless build return unless build.project.builds_enabled? @@ -282,6 +301,12 @@ module Gitlab REGISTRY_SCOPES end + + private + + def find_build_by_token(token) + ::Ci::Build.running.find_by_token(token) + end end end end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index a0b5cd868c3..66de52506ce 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -16,7 +16,7 @@ module Gitlab end def find_sessionless_user - find_user_from_access_token || find_user_from_rss_token + find_user_from_access_token || find_user_from_feed_token rescue Gitlab::Auth::AuthenticationError nil end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 4dc23f977da..c7993665421 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -25,13 +25,15 @@ module Gitlab current_request.env['warden']&.authenticate if verified_request? end - def find_user_from_rss_token - return unless current_request.path.ends_with?('.atom') || current_request.format.atom? + def find_user_from_feed_token + return unless rss_request? || ics_request? - token = current_request.params[:rss_token].presence + # NOTE: feed_token was renamed from rss_token but both needs to be supported because + # users might have already added the feed to their RSS reader before the rename + token = current_request.params[:feed_token].presence || current_request.params[:rss_token].presence return unless token - User.find_by_rss_token(token) || raise(UnauthorizedError) + User.find_by_feed_token(token) || raise(UnauthorizedError) end def find_user_from_access_token @@ -104,6 +106,14 @@ module Gitlab def current_request @current_request ||= ensure_action_dispatch_request(request) end + + def rss_request? + current_request.path.ends_with?('.atom') || current_request.format.atom? + end + + def ics_request? + current_request.path.ends_with?('.ics') || current_request.format.ics? + end end end end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb new file mode 100644 index 00000000000..22b0ac71920 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreJobArtifact + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + end + + def perform(start_id, stop_id) + FillFileStoreJobArtifact::JobArtifact + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb new file mode 100644 index 00000000000..d0816ae3ed5 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreLfsObject + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + def perform(start_id, stop_id) + FillFileStoreLfsObject::LfsObject + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb new file mode 100644 index 00000000000..94c65459a67 --- /dev/null +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillStoreUpload + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillStoreUpload::Upload + .where(store: nil) + .where(id: (start_id..stop_id)) + .update_all(store: 1) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 69b8a8fc68f..f34c11ca3c2 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -8,6 +8,9 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + ## # Populate pipeline with block argument of CreatePipelineService#execute. # diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb index cff924e27ef..8788af57a67 100644 --- a/lib/gitlab/ci/trace/http_io.rb +++ b/lib/gitlab/ci/trace/http_io.rb @@ -148,7 +148,7 @@ module Gitlab def get_chunk unless in_range? - response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http| + response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| http.request(request) end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index d7369060cc5..4c28489f45a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -85,7 +85,7 @@ module Gitlab .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') .group(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval})") .where(conditions) - .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) + .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end end end diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index bea78862757..0a88e052f60 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -7,7 +7,9 @@ module Gitlab end def value - @value ||= count_commits + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + @value ||= count_commits + end end private diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 74fed447289..3cac007a42c 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -143,8 +143,13 @@ module Gitlab .order(arel_table[column_sym]) ).as('row_id') - count = arel_table.from(arel_table.alias) - .project('COUNT(*)') + arel_from = if Gitlab.rails5? + arel_table.from.from(arel_table.alias) + else + arel_table.from(arel_table.alias) + end + + count = arel_from.project('COUNT(*)') .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) .as('ct') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 765fb0289a8..2820293ad5c 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -78,9 +78,12 @@ module Gitlab # Returns the raw diff content up to the given line index def diff_hunk(diff_line) - # Adding 2 because of the @@ diff header and Enum#take should consider - # an extra line, because we're passing an index. - raw_diff.each_line.take(diff_line.index + 2).join + diff_line_index = diff_line.index + # @@ (match) header is not kept if it's found in the top of the file, + # therefore we should keep an extra line on this scenario. + diff_line_index += 1 unless diff_lines.first.match? + + diff_lines.select { |line| line.index <= diff_line_index }.map(&:text).join("\n") end def old_sha diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0603141e441..a1e904cfef4 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -53,6 +53,10 @@ module Gitlab %w[match new-nonewline old-nonewline].include?(type) end + def match? + type == :match + end + def discussable? !meta? end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 8cf59fa8e28..8c72d00c1f3 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -138,8 +138,8 @@ module Gitlab def ee_branch_presence_check! ee_remotes.keys.each do |remote| - [ee_branch_prefix, ee_branch_suffix].each do |branch| - _, status = step("Fetching #{remote}/#{ee_branch_prefix}", %W[git fetch #{remote} #{branch}]) + [ce_branch, ee_branch_prefix, ee_branch_suffix].each do |branch| + _, status = step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) if status.zero? @ee_remote_with_branch = remote diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 8c082c0c336..f42088f980e 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -32,17 +32,13 @@ module Gitlab end def find_by_filename(query, except: []) - filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) - filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + filenames = search_filenames(query, except) - blob_refs = filenames.map { |filename| [ref, filename] } - blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024) - - blobs.map do |blob| + blobs(filenames).map do |blob| Gitlab::SearchResults::FoundBlob.new( id: blob.id, filename: blob.path, - basename: File.basename(blob.path), + basename: File.basename(blob.path, File.extname(blob.path)), ref: ref, startline: 1, data: blob.data, @@ -50,5 +46,21 @@ module Gitlab ) end end + + def search_filenames(query, except) + filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) + + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + filenames + end + + def blob_refs(filenames) + filenames.map { |filename| [ref, filename] } + end + + def blobs(filenames) + Gitlab::Git::Blob.batch(repository, blob_refs(filenames), blob_size_limit: 1024) + end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1a21625a322..4cbf20bfe76 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1185,15 +1185,17 @@ module Gitlab end def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) - with_repo_branch_commit(source_repository, source_branch_name) do |commit| - break unless commit - - Gitlab::Git::Compare.new( - self, - target_branch_name, - commit.sha, - straight: straight - ) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + with_repo_branch_commit(source_repository, source_branch_name) do |commit| + break unless commit + + Gitlab::Git::Compare.new( + self, + target_branch_name, + commit.sha, + straight: straight + ) + end end end @@ -1455,7 +1457,7 @@ module Gitlab gitaly_repository_client.cleanup if is_enabled && exists? end rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup - Rails.logger.error("Unable to clean repository on storage #{storage} with path #{path}: #{e.message}") + Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") Gitlab::Metrics.counter( :failed_repository_cleanup_total, 'Number of failed repository cleanup events' diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb index 2f611cef37b..391f0d70583 100644 --- a/lib/gitlab/git/storage/checker.rb +++ b/lib/gitlab/git/storage/checker.rb @@ -35,7 +35,7 @@ module Gitlab def initialize(storage, logger = Rails.logger) @storage = storage config = Gitlab.config.repositories.storages[@storage] - @storage_path = config.legacy_disk_path + @storage_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { config.legacy_disk_path } @logger = logger @hostname = Gitlab::Environment.hostname diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index e35054466ff..62427ac9cc4 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -22,13 +22,14 @@ module Gitlab def self.build(storage, hostname = Gitlab::Environment.hostname) config = Gitlab.config.repositories.storages[storage] - - if !config.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) - elsif !config.legacy_disk_path.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) - else - new(storage, hostname) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + if !config.present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) + elsif !config.legacy_disk_path.present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) + else + new(storage, hostname) + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 0abae70c443..550294916a4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -33,6 +33,11 @@ module Gitlab MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze + # We have a mechanism to let GitLab automatically opt in to all Gitaly + # features. We want to be able to exclude some features from automatic + # opt-in. That is what EXPLICIT_OPT_IN_REQUIRED is for. + EXPLICIT_OPT_IN_REQUIRED = [Gitlab::GitalyClient::StorageSettings::DISK_ACCESS_DENIED_FLAG].freeze + MUTEX = Mutex.new class << self @@ -234,7 +239,7 @@ module Gitlab when MigrationStatus::OPT_OUT true when MigrationStatus::OPT_IN - opt_into_all_features? + opt_into_all_features? && !EXPLICIT_OPT_IN_REQUIRED.include?(feature_name) else false end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1f5f88bf792..a4cc64de80d 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -78,7 +78,7 @@ module Gitlab def tree_entry(ref, path, limit = nil) request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, - revision: ref, + revision: encode_binary(ref), path: encode_binary(path), limit: limit.to_i ) diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 9a576e463e3..02fcb413abd 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -4,6 +4,8 @@ module Gitlab # where production code (app, config, db, lib) touches Git repositories # directly. class StorageSettings + extend Gitlab::TemporarilyAllow + DirectPathAccessError = Class.new(StandardError) InvalidConfigurationError = Class.new(StandardError) @@ -17,7 +19,21 @@ module Gitlab # This class will give easily recognizable NoMethodErrors Deprecated = Class.new - attr_reader :legacy_disk_path + MUTEX = Mutex.new + + DISK_ACCESS_DENIED_FLAG = :deny_disk_access + ALLOW_KEY = :allow_disk_access + + # If your code needs this method then your code needs to be fixed. + def self.allow_disk_access + temporarily_allow(ALLOW_KEY) { yield } + end + + def self.disk_access_denied? + !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG) + rescue + false # Err on the side of caution, don't break gitlab for people + end def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) @@ -34,6 +50,14 @@ module Gitlab @hash.fetch(:gitaly_address) end + def legacy_disk_path + if self.class.disk_access_denied? + raise DirectPathAccessError, "git disk access denied via the gitaly_#{DISK_ACCESS_DENIED_FLAG} feature" + end + + @legacy_disk_path + end + private def method_missing(m, *args, &block) diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 5482504e72e..22719e9a003 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -29,28 +29,28 @@ module Gitlab end def user - api.get("/api/v3/user").parsed + api.get("/api/v4/user").parsed end def issues(project_identifier) lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v3/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed + api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed end end def issue_comments(project_identifier, issue_id) lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v3/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed + api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed end end def project(id) - api.get("/api/v3/projects/#{id}").parsed + api.get("/api/v4/projects/#{id}").parsed end def projects lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v3/projects?per_page=#{PER_PAGE}&page=#{page}").parsed + api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed end end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e44d7934fda..195672f5a12 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -25,7 +25,7 @@ module Gitlab body = @formatter.author_line(issue["author"]["name"]) body += issue["description"] - comments = client.issue_comments(project_identifier, issue["id"]) + comments = client.issue_comments(project_identifier, issue["iid"]) if comments.any? body += @formatter.comments_header diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 3d0418261bb..430b8c10058 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -17,7 +17,7 @@ module Gitlab path: repo["path"], description: repo["description"], namespace_id: namespace.id, - visibility_level: repo["visibility_level"], + visibility_level: Gitlab::VisibilityLevel.level_value(repo["visibility"]), import_type: "gitlab", import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 0d31934347f..deaa14c8434 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -11,7 +11,7 @@ module Gitlab gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.shortcuts_path = help_page_path('shortcuts') + gon.shortcuts_path = Gitlab::Routing.url_helpers.help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb new file mode 100644 index 00000000000..0adac79f25a --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb @@ -0,0 +1,26 @@ +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log how much time an API request was queued by Workhorse. +module Gitlab + module GrapeLogging + module Loggers + class QueueDurationLogger < ::GrapeLogging::Loggers::Base + attr_accessor :start_time + + def before + @start_time = Time.now + end + + def parameters(request, _) + proxy_start = request.env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence + + return {} unless proxy_start && start_time + + # Time in milliseconds since gitlab-workhorse started the request + duration = (start_time.to_f * 1_000 - proxy_start.to_f / 1_000_000).round(2) + + { 'queue_duration': duration } + end + end + end + end +end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb new file mode 100644 index 00000000000..8aba42ccfce --- /dev/null +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -0,0 +1,71 @@ +module Gitlab + module HashedStorage + module RakeHelper + def self.batch_size + ENV.fetch('BATCH', 200).to_i + end + + def self.listing_limit + ENV.fetch('LIMIT', 500).to_i + end + + def self.project_id_batches(&block) + Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def self.legacy_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def self.hashed_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def self.relation_summary(relation_name, relation) + relation_count = relation.count + $stdout.puts "* Found #{relation_count} #{relation_name}".color(:green) + + relation_count + end + + def self.projects_list(relation_name, relation) + listing(relation_name, relation.with_route) do |project| + $stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red) + end + end + + def self.attachments_list(relation_name, relation) + listing(relation_name, relation) do |upload| + $stdout.puts " - #{upload.path} (id: #{upload.id})".color(:red) + end + end + + def self.listing(relation_name, relation) + relation_count = relation_summary(relation_name, relation) + return unless relation_count > 0 + + limit = listing_limit + + if relation_count > limit + $stdout.puts " ! Displaying first #{limit} #{relation_name}..." + end + + relation.find_each(batch_size: batch_size).with_index do |element, index| + yield element + + break if index + 1 >= limit + end + end + end + end +end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index 6e554383270..fcbf266b80b 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -77,7 +77,9 @@ module Gitlab end def storage_path(storage_name) - storages_paths[storage_name]&.legacy_disk_path + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + storages_paths[storage_name]&.legacy_disk_path + end end # All below test methods use shell commands to perform actions on storage volumes. diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 34169319b26..7c9fc5c15bb 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -7,14 +7,15 @@ module Gitlab new(*args).clean end - def initialize(relation_hash:, relation_class:) + def initialize(relation_hash:, relation_class:, excluded_keys: []) @relation_hash = relation_hash @relation_class = relation_class + @excluded_keys = excluded_keys end def clean @relation_hash.reject do |key, _value| - prohibited_key?(key) || !@relation_class.attribute_method?(key) + prohibited_key?(key) || !@relation_class.attribute_method?(key) || excluded_key?(key) end.except('id') end @@ -23,6 +24,12 @@ module Gitlab def prohibited_key?(key) key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) end + + def excluded_key?(key) + return false if @excluded_keys.empty? + + @excluded_keys.include?(key) + end end end end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 56042ddecbf..0c8fda07294 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -32,6 +32,10 @@ module Gitlab @methods[key].nil? ? {} : { methods: @methods[key] } end + def find_excluded_keys(klass_name) + @excluded_attributes[klass_name.to_sym]&.map(&:to_s) || [] + end + private def find_attributes_only(value) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 21ac7f7e0b6..da3667faf7a 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -28,6 +28,7 @@ project_tree: - project_members: - :user - merge_requests: + - :metrics - notes: - :author - events: @@ -98,8 +99,6 @@ excluded_attributes: - :import_jid - :created_at - :updated_at - - :import_jid - - :import_jid - :id - :star_count - :last_activity_at diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index d5590dde40f..4eb67fbe11e 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -88,16 +88,18 @@ module Gitlab end def project_params - @project_params ||= json_params.merge(override_params) + @project_params ||= begin + attrs = json_params.merge(override_params) + + # Cleaning all imported and overridden params + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs, + relation_class: Project, + excluded_keys: excluded_keys_for_relation(:project)) + end end def override_params - return {} unless params = @project.import_data&.data&.fetch('override_params', nil) - - @override_params ||= params.select do |key, _value| - Project.column_names.include?(key.to_s) && - !reader.project_tree[:except].include?(key.to_sym) - end + @override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} end def json_params @@ -171,7 +173,8 @@ module Gitlab relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), members_mapper: members_mapper, user: @user, - project: @restored_project) + project: @restored_project, + excluded_keys: excluded_keys_for_relation(relation)) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first @@ -192,6 +195,10 @@ module Gitlab def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end + + def excluded_keys_for_relation(relation) + @reader.attributes_finder.find_excluded_keys(relation) + end end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index eb7f5120592..e621c40fc7a 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -1,7 +1,7 @@ module Gitlab module ImportExport class Reader - attr_reader :tree + attr_reader :tree, :attributes_finder def initialize(shared:) @shared = shared diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 4a41a69840b..c5cf290f191 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -18,9 +18,10 @@ module Gitlab label: :project_label, custom_attributes: 'ProjectCustomAttribute', project_badges: 'Badge', + metrics: 'MergeRequest::Metrics', ci_cd_settings: 'ProjectCiCdSetting' }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze @@ -36,13 +37,30 @@ module Gitlab new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:) + def self.relation_class(relation_name) + # There are scenarios where the model is pluralized (e.g. + # MergeRequest::Metrics), and we don't want to force it to singular + # with #classify. + relation_name.to_s.classify.constantize + rescue NameError + relation_name.to_s.constantize + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: []) @relation_name = OVERRIDES[relation_sym] || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @user = user @project = project @imported_object_retries = 0 + + # Remove excluded keys from relation_hash + # We don't do this in the parsed_relation_hash because of the 'transformed attributes' + # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, + # in the create method that attribute is renamed to diff. And because diff is an excluded key, + # if we clean the excluded keys in the parsed_relation_hash, it will be removed + # from the object attributes and the export will fail. + @relation_hash.except!(*excluded_keys) end # Creates an object from an actual model with name "relation_sym" with params from @@ -187,7 +205,7 @@ module Gitlab end def relation_class - @relation_class ||= @relation_name.to_s.classify.constantize + @relation_class ||= self.class.relation_class(@relation_name) end def imported_object diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb index 3e54456e936..4e611e7f16c 100644 --- a/lib/gitlab/import_formatter.rb +++ b/lib/gitlab/import_formatter.rb @@ -9,6 +9,7 @@ module Gitlab end def author_line(author) + author ||= "Anonymous" "*Created by: #{author}*\n\n" end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 2e9b6e302f5..38bdc61d8ab 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -106,7 +106,8 @@ module Gitlab project_wiki = ProjectWiki.new(project) unless project_wiki.empty? - project_wiki.search_files(query) + ref = repository_ref || project.wiki.default_branch + Gitlab::WikiFileFinder.new(project, ref).find(query) else [] end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 4c83581c4b1..3c4ff5d1928 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -4,7 +4,7 @@ module Gitlab attach_to :active_record def sql(event) - unless event.payload[:name] == 'CACHE' + unless event.payload.fetch(:cached, event.payload[:name] == 'CACHE') Transaction.current&.increment end end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index bb778f37096..c82320a6036 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -1,13 +1,15 @@ module Gitlab module SlashCommands class Command < BaseCommand - COMMANDS = [ - Gitlab::SlashCommands::IssueShow, - Gitlab::SlashCommands::IssueNew, - Gitlab::SlashCommands::IssueSearch, - Gitlab::SlashCommands::IssueMove, - Gitlab::SlashCommands::Deploy - ].freeze + def self.commands + [ + Gitlab::SlashCommands::IssueShow, + Gitlab::SlashCommands::IssueNew, + Gitlab::SlashCommands::IssueSearch, + Gitlab::SlashCommands::IssueMove, + Gitlab::SlashCommands::Deploy + ] + end def execute command, match = match_command @@ -37,7 +39,7 @@ module Gitlab private def available_commands - COMMANDS.select do |klass| + self.class.commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb new file mode 100644 index 00000000000..880e55f71df --- /dev/null +++ b/lib/gitlab/temporarily_allow.rb @@ -0,0 +1,42 @@ +module Gitlab + module TemporarilyAllow + TEMPORARILY_ALLOW_MUTEX = Mutex.new + + def temporarily_allow(key) + temporarily_allow_add(key, 1) + yield + ensure + temporarily_allow_add(key, -1) + end + + def temporarily_allowed?(key) + if RequestStore.active? + temporarily_allow_request_store[key] > 0 + else + TEMPORARILY_ALLOW_MUTEX.synchronize do + temporarily_allow_ivar[key] > 0 + end + end + end + + private + + def temporarily_allow_ivar + @temporarily_allow ||= Hash.new(0) + end + + def temporarily_allow_request_store + RequestStore[:temporarily_allow] ||= Hash.new(0) + end + + def temporarily_allow_add(key, value) + if RequestStore.active? + temporarily_allow_request_store[key] += value + else + TEMPORARILY_ALLOW_MUTEX.synchronize do + temporarily_allow_ivar[key] += value + end + end + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index db97f65bd54..20be193ea0c 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -5,7 +5,7 @@ module Gitlab BlockedUrlError = Class.new(StandardError) class << self - def validate!(url, allow_localhost: false, allow_local_network: true, valid_ports: []) + def validate!(url, allow_localhost: false, allow_local_network: true, ports: [], protocols: []) return true if url.nil? begin @@ -18,7 +18,8 @@ module Gitlab return true if internal?(uri) port = uri.port || uri.default_port - validate_port!(port, valid_ports) if valid_ports.any? + validate_protocol!(uri.scheme, protocols) + validate_port!(port, ports) if ports.any? validate_user!(uri.user) validate_hostname!(uri.hostname) @@ -44,13 +45,19 @@ module Gitlab private - def validate_port!(port, valid_ports) + def validate_port!(port, ports) return if port.blank? # Only ports under 1024 are restricted return if port >= 1024 - return if valid_ports.include?(port) + return if ports.include?(port) - raise BlockedUrlError, "Only allowed ports are #{valid_ports.join(', ')}, and any over 1024" + raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024" + end + + def validate_protocol!(protocol, protocols) + if protocol.blank? || (protocols.any? && !protocols.include?(protocol)) + raise BlockedUrlError, "Only allowed protocols are #{protocols.join(', ')}" + end end def validate_user!(value) diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb index b9a75eaac63..529f7d6a8d6 100644 --- a/lib/gitlab/webpack/dev_server_middleware.rb +++ b/lib/gitlab/webpack/dev_server_middleware.rb @@ -15,6 +15,11 @@ module Gitlab def perform_request(env) if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}") + if relative_url_root = Rails.application.config.relative_url_root + env['SCRIPT_NAME'] = "" + env['REQUEST_PATH'].sub!(/\A#{Regexp.escape(relative_url_root)}/, '') + end + super(env) else @app.call(env) diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb new file mode 100644 index 00000000000..f97278f05cd --- /dev/null +++ b/lib/gitlab/wiki_file_finder.rb @@ -0,0 +1,23 @@ +module Gitlab + class WikiFileFinder < FileFinder + attr_reader :repository + + def initialize(project, ref) + @project = project + @ref = ref + @repository = project.wiki.repository + end + + private + + def search_filenames(query, except) + safe_query = Regexp.escape(query.tr(' ', '-')) + safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) + filenames = repository.ls_files(ref) + + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + filenames.grep(safe_query).first(BATCH_SIZE) + end + end +end |