diff options
author | DJ Mountney <david@twkie.net> | 2018-10-29 14:37:19 -0700 |
---|---|---|
committer | DJ Mountney <david@twkie.net> | 2018-10-29 14:37:19 -0700 |
commit | 06b6daacb15b92b04e05538b37aadfdb04fc5a4b (patch) | |
tree | 0500cf47f7d8d2e6bc7ed53cbf9ad957f07fb31a /lib | |
parent | c847f172d25efc211045c363f4e55402ad250c09 (diff) | |
parent | 45b61a9ece48550f51432c8cca7de7e1a298ca08 (diff) | |
download | gitlab-ce-06b6daacb15b92b04e05538b37aadfdb04fc5a4b.tar.gz |
Merge remote-tracking branch 'origin/master' into dev-master
Diffstat (limited to 'lib')
24 files changed, 434 insertions, 84 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 18c30723d73..9f7be27b047 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -160,13 +160,27 @@ module API # (fixed in https://github.com/rails/rails/pull/25976). project.tags.map(&:name).sort end + expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url + + expose :license_url, if: :license do |project| + license = project.repository.license_blob + + if license + Gitlab::Routing.url_helpers.project_blob_url(project, File.join(project.default_branch, license.path)) + end + end + + expose :license, with: 'API::Entities::LicenseBasic', if: :license do |project| + project.repository.license + end + expose :avatar_url do |project, options| project.avatar_url(only_path: false) end + expose :star_count, :forks_count expose :last_activity_at - expose :namespace, using: 'API::Entities::NamespaceBasic' expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes @@ -1208,11 +1222,14 @@ module API expose :deployable, using: Entities::Job end - class License < Grape::Entity + class LicenseBasic < Grape::Entity expose :key, :name, :nickname - expose :popular?, as: :popular expose :url, as: :html_url expose(:source_url) { |license| license.meta['source'] } + end + + class License < LicenseBasic + expose :popular?, as: :popular expose(:description) { |license| license.meta['description'] } expose(:conditions) { |license| license.meta['conditions'] } expose(:permissions) { |license| license.meta['permissions'] } diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index 23b1cd1ad45..1058f4e8a5e 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -10,8 +10,21 @@ module API raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) end end + + class IntegerNoneAny < Grape::Validations::Base + def validate_param!(attr_name, params) + value = params[attr_name] + + return if value.is_a?(Integer) || + [IssuableFinder::FILTER_NONE, IssuableFinder::FILTER_ANY].include?(value.to_s.downcase) + + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], + message: "should be an integer, 'None' or 'Any'" + end + end end end end Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) +Grape::Validations.register_validator(:integer_none_any, ::API::Helpers::CustomValidators::IntegerNoneAny) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 4dd6b19e353..ae40b5f7557 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -65,6 +65,8 @@ module API result rescue Gitlab::GitAccess::UnauthorizedError => e break response_with_status(code: 401, success: false, message: e.message) + rescue Gitlab::GitAccess::TimeoutError => e + break response_with_status(code: 503, success: false, message: e.message) rescue Gitlab::GitAccess::NotFoundError => e break response_with_status(code: 404, success: false, message: e.message) end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 25d78053c88..405fc30a2ed 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -40,7 +40,8 @@ module API optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' - optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' + optional :assignee_id, types: [Integer, String], integer_none_any: true, + desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 440d94ae186..a617efaaa4c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -89,7 +89,8 @@ module API optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' - optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' + optional :assignee_id, types: [Integer, String], integer_none_any: true, + desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ae2d327e45b..0a914f9012e 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -114,7 +114,8 @@ module API options = options.reverse_merge( with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, statistics: params[:statistics], - current_user: current_user + current_user: current_user, + license: false ) options[:with] = Entities::BasicProjectDetails if params[:simple] @@ -230,13 +231,17 @@ module API params do use :statistics_params use :with_custom_attributes + + optional :license, type: Boolean, default: false, + desc: 'Include project license data' end get ":id" do options = { with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, current_user: current_user, user_can_admin_project: can?(current_user, :admin_project, user_project), - statistics: params[:statistics] + statistics: params[:statistics], + license: params[:license] } project, options = with_custom_attributes(user_project, options) diff --git a/lib/api/runner.rb b/lib/api/runner.rb index d8768a54986..2f15f3a7d76 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -142,8 +142,7 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :trace, type: String, desc: %q(Job's full trace) optional :state, type: String, desc: %q(Job's status: success, failed) - optional :failure_reason, type: String, values: CommitStatus.failure_reasons.keys, - desc: %q(Job's failure_reason) + optional :failure_reason, type: String, desc: %q(Job's failure_reason) end put '/:id' do job = authenticate_job! diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b369b9e7600..dfbb83f7bb9 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -42,7 +42,7 @@ module Gitlab end def self.cache_key_for_project(project) - "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:projects/#{project.id}/pipeline_status" + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:projects/#{project.id}/pipeline_status/#{project.commit&.sha}" end def self.update_for_pipeline(pipeline) @@ -84,9 +84,7 @@ module Gitlab def load_from_project return unless commit - self.sha = commit.sha - self.status = commit.status - self.ref = project.default_branch + self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch end # We only cache the status for the HEAD commit of a project @@ -104,6 +102,8 @@ module Gitlab def load_from_cache Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) + + self.status = nil if self.status.empty? end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 49e7f7e1fd7..074afe9c412 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -18,11 +18,24 @@ module Gitlab lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze - attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name + LOG_MESSAGES = { + push_checks: "Checking if you are allowed to push...", + delete_default_branch_check: "Checking if default branch is being deleted...", + protected_branch_checks: "Checking if you are force pushing to a protected branch...", + protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", + protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch...", + tag_checks: "Checking if you are allowed to change existing tags...", + protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag...", + lfs_objects_exist_check: "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...", + commits_check_file_paths_validation: "Validating commits' file paths...", + commits_check: "Validating commit contents..." + }.freeze + + attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name, :logger def initialize( change, user_access:, project:, skip_authorization: false, - skip_lfs_integrity_check: false, protocol: + skip_lfs_integrity_check: false, protocol:, logger: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @@ -32,6 +45,9 @@ module Gitlab @skip_authorization = skip_authorization @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol + + @logger = logger + @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") end def exec(skip_commits_check: false) @@ -49,26 +65,32 @@ module Gitlab protected def push_checks - unless can_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + logger.log_timed(LOG_MESSAGES[__method__]) do + unless can_push? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + end end end def branch_checks return unless branch_name - if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do + if deletion? && branch_name == project.default_branch + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + end end protected_branch_checks end def protected_branch_checks - return unless ProtectedBranch.protected?(project, branch_name) + logger.log_timed(LOG_MESSAGES[__method__]) do + return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks - if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + if forced_push? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + end end if deletion? @@ -79,23 +101,27 @@ module Gitlab end def protected_branch_deletion_checks - unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] - end + logger.log_timed(LOG_MESSAGES[__method__]) do + unless user_access.can_delete_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + end - unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + unless updated_from_web? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + end end end def protected_branch_push_checks - if matching_merge_request? - unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] - end - else - unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + logger.log_timed(LOG_MESSAGES[__method__]) do + if matching_merge_request? + unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + end + else + unless user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + end end end end @@ -103,21 +129,25 @@ module Gitlab def tag_checks return unless tag_name - if tag_exists? && user_access.cannot_do_action?(:admin_project) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + logger.log_timed(LOG_MESSAGES[__method__]) do + if tag_exists? && user_access.cannot_do_action?(:admin_project) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + end end protected_tag_checks end def protected_tag_checks - return unless ProtectedTag.protected?(project, tag_name) + logger.log_timed(LOG_MESSAGES[__method__]) do + return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? - unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + unless user_access.can_create_tag?(tag_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + end end end @@ -125,14 +155,20 @@ module Gitlab return if deletion? || newrev.nil? return unless should_run_commit_validations? - # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - commits.each do |commit| - commit_check.validate(commit, validations_for_commit(commit)) + logger.log_timed(LOG_MESSAGES[__method__]) do + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + logger.check_timeout_reached + + commit_check.validate(commit, validations_for_commit(commit)) + end end end - commit_check.validate_file_paths + logger.log_timed(LOG_MESSAGES[:commits_check_file_paths_validation]) do + commit_check.validate_file_paths + end end # Method overwritten in EE to inject custom validations @@ -194,10 +230,12 @@ module Gitlab end def lfs_objects_exist_check - lfs_check = Checks::LfsIntegrity.new(project, newrev) + logger.log_timed(LOG_MESSAGES[__method__]) do + lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) - if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] + if lfs_check.objects_missing? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] + end end end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index fa3dc1808df..1652d5a30a4 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -3,9 +3,10 @@ module Gitlab module Checks class LfsIntegrity - def initialize(project, newrev) + def initialize(project, newrev, time_left) @project = project @newrev = newrev + @time_left = time_left end # rubocop: disable CodeReuse/ActiveRecord @@ -13,7 +14,7 @@ module Gitlab return false unless @newrev && @project.lfs_enabled? new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev) - .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT) + .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT, dynamic_timeout: @time_left) return false unless new_lfs_pointers.present? diff --git a/lib/gitlab/checks/timed_logger.rb b/lib/gitlab/checks/timed_logger.rb new file mode 100644 index 00000000000..f365e0a43f6 --- /dev/null +++ b/lib/gitlab/checks/timed_logger.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class TimedLogger + TimeoutError = Class.new(StandardError) + + attr_reader :start_time, :header, :log, :timeout + + def initialize(start_time: Time.now, log: [], header: "", timeout:) + @start_time = start_time + @timeout = timeout + @header = header + @log = log + end + + # Adds trace of method being tracked with + # the correspondent time it took to run it. + # We make use of the start default argument + # on unit tests related to this method + # + def log_timed(log_message, start = Time.now) + check_timeout_reached + + timed = true + + yield + + append_message(log_message + time_suffix_message(start: start)) + rescue GRPC::DeadlineExceeded, TimeoutError + args = { cancelled: true } + args[:start] = start if timed + + append_message(log_message + time_suffix_message(args)) + + raise TimeoutError + end + + def check_timeout_reached + return unless time_expired? + + raise TimeoutError + end + + def time_left + (start_time + timeout.seconds) - Time.now + end + + def full_message + header + log.join("\n") + end + + # We always want to append in-place on the log + def append_message(message) + log << message + end + + private + + def time_expired? + time_left <= 0 + end + + def time_suffix_message(cancelled: false, start: nil) + return " (#{elapsed_time(start)}ms)" unless cancelled + + if start + " (cancelled after #{elapsed_time(start)}ms)" + else + " (cancelled)" + end + end + + def elapsed_time(start) + to_ms(Time.now - start) + end + + def to_ms(elapsed) + (elapsed.to_f * 1000).round(2) + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 50b0d044265..4babc23a495 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -11,7 +11,8 @@ module Gitlab runner_system_failure: 'runner system failure', missing_dependency_failure: 'missing dependency failure', runner_unsupported: 'unsupported runner', - stale_schedule: 'stale schedule' + stale_schedule: 'stale schedule', + job_execution_timeout: 'job execution timeout' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 6fa59e41d20..db48b187e5e 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -210,7 +210,7 @@ container_scanning: refs: - branches variables: - - $GITLAB_FEATURES =~ /\bsast_container\b/ + - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ except: variables: - $CONTAINER_SCANNING_DISABLED diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb new file mode 100644 index 00000000000..b05dca409d1 --- /dev/null +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + # + # LifecycleEvents lets Rails initializers register application startup hooks + # that are sensitive to forking. For example, to defer the creation of + # watchdog threads. This lets us abstract away the Unix process + # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc. + # + # We have three lifecycle events. + # + # - before_fork (only in forking processes) + # - worker_start + # - before_master_restart (only in forking processes) + # + # Blocks will be executed in the order in which they are registered. + # + class LifecycleEvents + class << self + # + # Hook registration methods (called from initializers) + # + def on_worker_start(&block) + if in_clustered_environment? + # Defer block execution + (@worker_start_hooks ||= []) << block + else + yield + end + end + + def on_before_fork(&block) + return unless in_clustered_environment? + + # Defer block execution + (@before_fork_hooks ||= []) << block + end + + def on_master_restart(&block) + return unless in_clustered_environment? + + # Defer block execution + (@master_restart_hooks ||= []) << block + end + + # + # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) + # + def do_worker_start + @worker_start_hooks&.each do |block| + block.call + end + end + + def do_before_fork + @before_fork_hooks&.each do |block| + block.call + end + end + + def do_master_restart + @master_restart_hooks && @master_restart_hooks.each do |block| + block.call + end + end + + # Puma doesn't use singletons (which is good) but + # this means we need to pass through whether the + # puma server is running in single mode or cluster mode + def set_puma_options(options) + @puma_options = options + end + + private + + def in_clustered_environment? + # Sidekiq doesn't fork + return false if Sidekiq.server? + + # Unicorn always forks + return true if defined?(::Unicorn) + + # Puma sometimes forks + return true if in_clustered_puma? + + # Default assumption is that we don't fork + false + end + + def in_clustered_puma? + return false unless defined?(::Puma) + + @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0 + end + end + end + end +end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb new file mode 100644 index 00000000000..331c39f7d6b --- /dev/null +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + class PumaWorkerKillerInitializer + def self.start(puma_options, puma_per_worker_max_memory_mb: 650) + require 'puma_worker_killer' + + PumaWorkerKiller.config do |config| + # Note! ram is expressed in megabytes (whereas GITLAB_UNICORN_MEMORY_MAX is in bytes) + # Importantly RAM is for _all_workers (ie, the cluster), + # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX + worker_count = puma_options[:workers] || 1 + config.ram = worker_count * puma_per_worker_max_memory_mb + + config.frequency = 20 # seconds + + # We just want to limit to a fixed maximum, unrelated to the total amount + # of available RAM. + config.percent_usage = 0.98 + + # Ideally we'll never hit the maximum amount of memory. If so the worker + # is restarted already, thus periodically restarting workers shouldn't be + # needed. + config.rolling_restart_frequency = false + end + + PumaWorkerKiller.start + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index c819bffdfd6..5ed6427072a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -73,7 +73,7 @@ module Gitlab # re-running the contributed projects query in each union is expensive, so # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short - @contributed_project_ids ||= projects.uniq.pluck(:id) + @contributed_project_ids ||= projects.distinct.pluck(:id) authed_projects = Project.where(id: @contributed_project_ids) .with_feature_available_for_user(feature, current_user) .reorder(nil) diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index f0fab1e76a3..d7148165408 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -6,8 +6,8 @@ module Gitlab @newrev = newrev end - def new_pointers(object_limit: nil, not_in: nil) - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) + def new_pointers(object_limit: nil, not_in: nil, dynamic_timeout: nil) + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in, dynamic_timeout) end def all_pointers diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 827c04ae035..802fa65dd63 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -9,6 +9,7 @@ module Gitlab UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) + TimeoutError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) ERROR_MESSAGES = { @@ -26,11 +27,18 @@ module Gitlab cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze + INTERNAL_TIMEOUT = 50.seconds.freeze + LOG_HEADER = <<~MESSAGE + Push operation timed out + + Timing information for debugging purposes: + MESSAGE + DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze PUSH_COMMANDS = %w{git-receive-pack}.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes, :logger def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @@ -44,6 +52,7 @@ module Gitlab end def check(cmd, changes) + @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER) @changes = changes check_protocol! @@ -269,14 +278,19 @@ module Gitlab end def check_single_change_access(change, skip_lfs_integrity_check: false) - Checks::ChangeAccess.new( + change_access = Checks::ChangeAccess.new( change, user_access: user_access, project: project, skip_authorization: deploy_key?, skip_lfs_integrity_check: skip_lfs_integrity_check, - protocol: protocol - ).exec + protocol: protocol, + logger: logger + ) + + change_access.exec + rescue Checks::TimedLogger::TimeoutError + raise TimeoutError, logger.full_message end def deploy_key diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index e731e654f3c..cf2329e489d 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -11,8 +11,8 @@ module Gitlab @changes = deserialize_changes(changes) end - def identify(revision) - super(identifier, project, revision) + def identify + super(identifier) end def changes_refs diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 1840bf45154..086ce31e678 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -72,7 +72,7 @@ module Gitlab GitalyClient::BlobsStitcher.new(response) end - def get_new_lfs_pointers(revision, limit, not_in) + def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) request = Gitaly::GetNewLFSPointersRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), @@ -85,7 +85,20 @@ module Gitlab request.not_in_refs += not_in end - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + timeout = + if dynamic_timeout + [dynamic_timeout, GitalyClient.medium_timeout].min + else + GitalyClient.medium_timeout + end + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_new_lfs_pointers, + request, + timeout: timeout + ) map_lfs_pointers(response) end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index 0014ce2689b..41004408dec 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -6,7 +6,7 @@ module Gitlab def call(severity, datetime, _, data) time = data.delete :time - data[:params] = utf8_encode_values(data[:params]) if data.has_key?(:params) + data[:params] = process_params(data) attributes = { time: datetime.utc.iso8601(3), @@ -20,6 +20,14 @@ module Gitlab private + def process_params(data) + return [] unless data.has_key?(:params) + + data[:params] + .each_pair + .map { |k, v| { key: k, value: utf8_encode_values(v) } } + end + def utf8_encode_values(data) case data when Hash diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 28a9fe29465..d5f94ad04f1 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -1,13 +1,11 @@ # frozen_string_literal: true # Detect user based on identifier like -# key-13 or user-36 or last commit +# key-13 or user-36 module Gitlab module Identifier - def identify(identifier, project = nil, newrev = nil) - if identifier.blank? - identify_using_commit(project, newrev) - elsif identifier =~ /\Auser-\d+\Z/ + def identify(identifier) + if identifier =~ /\Auser-\d+\Z/ # git push over http identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ @@ -16,19 +14,6 @@ module Gitlab end end - # Tries to identify a user based on a commit SHA. - def identify_using_commit(project, ref) - return if project.nil? && ref.nil? - - commit = project.commit(ref) - - return if !commit || !commit.author_email - - identify_with_cache(:email, commit.author_email) do - commit.author - end - end - # Tries to identify a user based on a user identifier (e.g. "user-123"). # rubocop: disable CodeReuse/ActiveRecord def identify_using_user(identifier) diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb new file mode 100644 index 00000000000..b00244a6e04 --- /dev/null +++ b/lib/gitlab/patch/draw_route.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# We're patching `ActionDispatch::Routing::Mapper` in +# config/initializers/routing_draw.rb +module Gitlab + module Patch + module DrawRoute + RoutesNotFound = Class.new(StandardError) + + def draw(routes_name) + drawn_any = draw_ce(routes_name) | draw_ee(routes_name) + + drawn_any || raise(RoutesNotFound.new("Cannot find #{routes_name}")) + end + + def draw_ce(routes_name) + draw_route(route_path("config/routes/#{routes_name}.rb")) + end + + def draw_ee(_) + true + end + + def route_path(routes_name) + Rails.root.join(routes_name) + end + + def draw_route(path) + if File.exist?(path) + instance_eval(File.read(path)) + true + else + false + end + end + end + end +end diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake index c66a2a263dc..0459de27c96 100644 --- a/lib/tasks/gitlab/ldap.rake +++ b/lib/tasks/gitlab/ldap.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :ldap do desc 'GitLab | LDAP | Rename provider' - task :rename_provider, [:old_provider, :new_provider] => :environment do |_, args| + task :rename_provider, [:old_provider, :new_provider] => :gitlab_environment do |_, args| old_provider = args[:old_provider] || prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue)) new_provider = args[:new_provider] || |