diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/broadcast_messages.rb | 9 | ||||
-rw-r--r-- | lib/api/helpers/internal_helpers.rb | 14 | ||||
-rw-r--r-- | lib/constraints/project_url_constrainer.rb | 2 | ||||
-rw-r--r-- | lib/feature/gitaly.rb | 1 | ||||
-rw-r--r-- | lib/gitaly/server.rb | 32 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/server_service.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/gon_helper.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/import_export/import_failure_service.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_restorer.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/middleware/read_only/controller.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_config.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_config/worker.rb | 15 | ||||
-rw-r--r-- | lib/tasks/gitlab/sidekiq.rake | 68 | ||||
-rw-r--r-- | lib/tasks/lint.rake | 10 |
16 files changed, 212 insertions, 46 deletions
diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 994e12445b7..48d4f1a0a63 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -4,9 +4,6 @@ module API class BroadcastMessages < Grape::API include PaginationParams - before { authenticate! } - before { authenticated_as_admin! } - resource :broadcast_messages do helpers do def find_message @@ -40,6 +37,8 @@ module API optional :target_path, type: String, desc: 'Target path' end post do + authenticated_as_admin! + message = BroadcastMessage.create(declared_params(include_missing: false)) if message.persisted? @@ -76,6 +75,8 @@ module API optional :target_path, type: String, desc: 'Target path' end put ':id' do + authenticated_as_admin! + message = find_message if message.update(declared_params(include_missing: false)) @@ -93,6 +94,8 @@ module API requires :id, type: Integer, desc: 'Broadcast message ID' end delete ':id' do + authenticated_as_admin! + message = find_message destroy_conditionally!(message) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index c8e96c7c5db..cc9d45dcae4 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -104,14 +104,12 @@ module API # rubocop:disable Gitlab/ModuleWithInstanceVariables def set_project - if params[:gl_repository] - @project, @repo_type = Gitlab::GlRepository.parse(params[:gl_repository]) - @redirected_path = nil - elsif params[:project] - @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse(params[:project]) - else - @project, @repo_type, @redirected_path = nil, nil, nil - end + @project, @repo_type, @redirected_path = + if params[:gl_repository] + Gitlab::GlRepository.parse(params[:gl_repository]) + elsif params[:project] + Gitlab::RepoPath.parse(params[:project]) + end end # rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index d41490d2ebd..3e9cf2ab320 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,7 +4,7 @@ module Constraints class ProjectUrlConstrainer def matches?(request, existence_check: true) namespace_path = request.params[:namespace_id] - project_path = request.params[:project_id] || request.params[:id] + project_path = request.params[:project_id] || request.params[:id] || request.params[:repository_id] full_path = [namespace_path, project_path].join('/') return false unless ProjectPathValidator.valid_path?(full_path) diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index c225ca56c55..96062ae87bc 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -9,7 +9,6 @@ class Feature %w[ cache_invalidator inforef_uploadpack_cache - get_tag_messages_go filter_shas_with_signatures_go commit_without_batch_check ].freeze diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 64ab5db4fcd..89a836e629f 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -53,6 +53,20 @@ module Gitaly storage_status&.fs_type end + def disk_used + disk_statistics_storage_status&.used + end + + def disk_available + disk_statistics_storage_status&.available + end + + # Simple convenience method for when obtaining both used and available + # statistics at once is preferred. + def disk_stats + disk_statistics_storage_status + end + def address Gitlab::GitalyClient.address(@storage) rescue RuntimeError => e @@ -65,6 +79,10 @@ module Gitaly @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage } end + def disk_statistics_storage_status + @disk_statistics_storage_status ||= disk_statistics.storage_statuses.find { |s| s.storage_name == storage } + end + def matches_sha? match = server_version.match(SHA_VERSION_REGEX) return false unless match @@ -76,7 +94,19 @@ module Gitaly @info ||= begin Gitlab::GitalyClient::ServerService.new(@storage).info - rescue GRPC::Unavailable, GRPC::DeadlineExceeded + rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex + Gitlab::ErrorTracking.track_exception(ex) + # This will show the server as being out of date + Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) + end + end + + def disk_statistics + @disk_statistics ||= + begin + Gitlab::GitalyClient::ServerService.new(@storage).disk_statistics + rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex + Gitlab::ErrorTracking.track_exception(ex) # This will show the server as being out of date Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 262a1ef653f..4eb1ccf32ba 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -432,10 +432,7 @@ module Gitlab end def self.filesystem_id(storage) - response = Gitlab::GitalyClient::ServerService.new(storage).info - storage_status = response.storage_statuses.find { |status| status.storage_name == storage } - - storage_status&.filesystem_id + Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id end def self.filesystem_id_from_disk(storage) @@ -446,6 +443,14 @@ module Gitlab nil end + def self.filesystem_disk_available(storage) + Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available + end + + def self.filesystem_disk_used(storage) + Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used + end + def self.timeout(timeout_name) Gitlab::CurrentSettings.current_application_settings[timeout_name] end diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb index 0ade6942db9..36bda67c26e 100644 --- a/lib/gitlab/gitaly_client/server_service.rb +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -13,6 +13,24 @@ module Gitlab def info GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new, timeout: GitalyClient.fast_timeout) end + + def disk_statistics + GitalyClient.call(@storage, :server_service, :disk_statistics, Gitaly::DiskStatisticsRequest.new, timeout: GitalyClient.fast_timeout) + end + + def storage_info + storage_specific(info) + end + + def storage_disk_statistics + storage_specific(disk_statistics) + end + + private + + def storage_specific(response) + response.storage_statuses.find { |status| status.storage_name == @storage } + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2e27e954e79..3db6c3b51c0 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -43,6 +43,9 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:snippets_vue, default_enabled: false) + push_frontend_feature_flag(:monaco_snippets, default_enabled: false) + push_frontend_feature_flag(:monaco_blobs, default_enabled: false) + push_frontend_feature_flag(:monaco_ci, default_enabled: false) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/import_export/import_failure_service.rb b/lib/gitlab/import_export/import_failure_service.rb index eeaf10870c8..d4eca551b49 100644 --- a/lib/gitlab/import_export/import_failure_service.rb +++ b/lib/gitlab/import_export/import_failure_service.rb @@ -12,9 +12,14 @@ module Gitlab @association = importable.association(:import_failures) end - def with_retry(relation_key, relation_index) + def with_retry(action:, relation_key: nil, relation_index: nil) on_retry = -> (exception, retry_count, *_args) do - log_import_failure(relation_key, relation_index, exception, retry_count) + log_import_failure( + source: action, + relation_key: relation_key, + relation_index: relation_index, + exception: exception, + retry_count: retry_count) end Retriable.with_context(:relation_import, on_retry: on_retry) do @@ -22,8 +27,9 @@ module Gitlab end end - def log_import_failure(relation_key, relation_index, exception, retry_count = 0) + def log_import_failure(source:, relation_key: nil, relation_index: nil, exception:, retry_count: 0) extra = { + source: source, relation_key: relation_key, relation_index: relation_index, retry_count: retry_count diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index e598cfc143e..c4ac6a3a3f2 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -21,7 +21,9 @@ module Gitlab RelationRenameService.rename(@tree_hash) if relation_tree_restorer.restore - @project.merge_requests.set_latest_merge_request_diff_ids! + import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do + @project.merge_requests.set_latest_merge_request_diff_ids! + end true else @@ -72,6 +74,10 @@ module Gitlab def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end + + def import_failure_service + @import_failure_service ||= ImportFailureService.new(@project) + end end end end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 44cf90fb86a..3606e1c5bd9 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -73,13 +73,17 @@ module Gitlab relation_object.assign_attributes(importable_class_sym => @importable) - import_failure_service.with_retry(relation_key, relation_index) do + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do relation_object.save! end save_id_mapping(relation_key, data_hash, relation_object) rescue => e - import_failure_service.log_import_failure(relation_key, relation_index, e) + import_failure_service.log_import_failure( + source: 'process_relation_item!', + relation_key: relation_key, + relation_index: relation_index, + exception: e) end def import_failure_service diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index c749816cf6a..2cf2a81f812 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -12,12 +12,12 @@ module Gitlab ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance' WHITELISTED_GIT_ROUTES = { - 'projects/git_http' => %w{git_upload_pack git_receive_pack} + 'repositories/git_http' => %w{git_upload_pack git_receive_pack} }.freeze WHITELISTED_GIT_LFS_ROUTES = { - 'projects/lfs_api' => %w{batch}, - 'projects/lfs_locks_api' => %w{verify create unlock} + 'repositories/lfs_api' => %w{batch}, + 'repositories/lfs_locks_api' => %w{verify create unlock} }.freeze WHITELISTED_GIT_REVISION_ROUTES = { diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index c96212f27a7..c8719023d40 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -6,6 +6,7 @@ module Gitlab module SidekiqConfig FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml' EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml' + SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml' QUEUE_CONFIG_PATHS = [ FOSS_QUEUE_CONFIG_PATH, @@ -13,11 +14,19 @@ module Gitlab ].compact.freeze # For queues that don't have explicit workers - default and mailers - DummyWorker = Struct.new(:queue) + DummyWorker = Struct.new(:queue, :weight) do + def queue_namespace + nil + end + + def get_weight + weight + end + end DEFAULT_WORKERS = [ - Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default'), ee: false), - Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers'), ee: false) + Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default', 1), ee: false), + Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers', 2), ee: false) ].freeze class << self @@ -30,7 +39,7 @@ module Gitlab def config_queues @config_queues ||= begin - config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml')) + config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH)) config[:queues].map(&:first) end end @@ -65,6 +74,28 @@ module Gitlab Gitlab.ee? && ee_workers != YAML.safe_load(File.read(EE_QUEUE_CONFIG_PATH)) end + def queues_for_sidekiq_queues_yml + namespaces_with_equal_weights = + workers + .group_by(&:queue_namespace) + .map(&:last) + .select { |workers| workers.map(&:get_weight).uniq.count == 1 } + .map(&:first) + + namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set + remaining_queues = workers.reject { |worker| namespaces.include?(worker.queue_namespace) } + + (namespaces_with_equal_weights.map(&:namespace_and_weight) + + remaining_queues.map(&:queue_and_weight)).sort + end + + def sidekiq_queues_yml_outdated? + # YAML.load is OK here as we control the file contents + config_queues = YAML.load(File.read(SIDEKIQ_QUEUES_PATH))[:queues] # rubocop:disable Security/YAMLLoad + + queues_for_sidekiq_queues_yml != config_queues + end + private def find_workers(root, ee:) diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index 313d9b17b16..ac94bab9a8f 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -7,8 +7,9 @@ module Gitlab attr_reader :klass delegate :feature_category_not_owned?, :get_feature_category, - :get_worker_resource_boundary, :latency_sensitive_worker?, - :queue, :worker_has_external_dependencies?, + :get_weight, :get_worker_resource_boundary, + :latency_sensitive_worker?, :queue, :queue_namespace, + :worker_has_external_dependencies?, to: :klass def initialize(klass, ee:) @@ -35,7 +36,7 @@ module Gitlab # Put namespaced queues first def to_sort - [queue.include?(':') ? 0 : 1, queue] + [queue_namespace ? 0 : 1, queue] end # YAML representation @@ -46,6 +47,14 @@ module Gitlab def to_yaml queue end + + def namespace_and_weight + [queue_namespace, get_weight] + end + + def queue_and_weight + [queue, get_weight] + end end end end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 15c6c930386..eb3de195626 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -4,8 +4,13 @@ return if Rails.env.production? namespace :gitlab do namespace :sidekiq do + def write_yaml(path, banner, object) + File.write(path, banner + YAML.dump(object)) + end + namespace :all_queues_yml do - def write_yaml(path, object) + desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' + task generate: :environment do banner = <<~BANNER # This file is generated automatically by # bin/rake gitlab:sidekiq:all_queues_yml:generate @@ -13,17 +18,12 @@ namespace :gitlab do # Do not edit it manually! BANNER - File.write(path, banner + YAML.dump(object)) - end - - desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' - task generate: :environment do foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml - write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, foss_workers) + write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, banner, foss_workers) if Gitlab.ee? - write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, ee_workers) + write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, banner, ee_workers) end end @@ -44,5 +44,57 @@ namespace :gitlab do end end end + + namespace :sidekiq_queues_yml do + desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions' + task generate: :environment do + banner = <<~BANNER + # This file is generated automatically by + # bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate + # + # Do not edit it manually! + # + # This configuration file should be exclusively used to set queue settings for + # Sidekiq. Any other setting should be specified using the Sidekiq CLI or the + # Sidekiq Ruby API (see config/initializers/sidekiq.rb). + # + # All the queues to process and their weights. Every queue _must_ have a weight + # defined. + # + # The available weights are as follows + # + # 1: low priority + # 2: medium priority + # 3: high priority + # 5: _super_ high priority, this should only be used for _very_ important queues + # + # As per http://stackoverflow.com/a/21241357/290102 the formula for calculating + # the likelihood of a job being popped off a queue (given all queues have work + # to perform) is: + # + # chance = (queue weight / total weight of all queues) * 100 + BANNER + + queues_and_weights = Gitlab::SidekiqConfig.queues_for_sidekiq_queues_yml + + write_yaml(Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH, banner, queues: queues_and_weights) + end + + desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions' + task check: :environment do + if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated? + raise <<~MSG + Changes in worker queues found, please update the metadata by running: + + bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate + + Then commit and push the changes from: + + - #{Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH} + + MSG + end + end + end end end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 91e7031744a..7a4d09bb6d4 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -38,11 +38,13 @@ unless Rails.env.production? ] if Gitlab.ee? - # This task will fail on CE installations (e.g. gitlab-org/gitlab-foss) - # since it will detect strings in the locale files that do not exist in - # the source files. To work around this we will only enable this task on - # EE installations. + # These tasks will fail on FOSS installations + # (e.g. gitlab-org/gitlab-foss) since they test against a single + # file that is generated by an EE installation, which can + # contain values that a FOSS installation won't find. To work + # around this we will only enable this task on EE installations. tasks << 'gettext:updated_check' + tasks << 'gitlab:sidekiq:sidekiq_queues_yml:check' end tasks.each do |task| |