diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
commit | 8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781 (patch) | |
tree | a77e7fe7a93de11213032ed4ab1f33a3db51b738 /app/workers/concerns | |
parent | 00b35af3db1abfe813a778f643dad221aad51fca (diff) | |
download | gitlab-ce-8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781.tar.gz |
Add latest changes from gitlab-org/gitlab@13-1-stable-ee
Diffstat (limited to 'app/workers/concerns')
-rw-r--r-- | app/workers/concerns/application_worker.rb | 27 | ||||
-rw-r--r-- | app/workers/concerns/gitlab/github_import/queue.rb | 11 | ||||
-rw-r--r-- | app/workers/concerns/gitlab/jira_import/import_worker.rb | 2 | ||||
-rw-r--r-- | app/workers/concerns/project_import_options.rb | 2 | ||||
-rw-r--r-- | app/workers/concerns/reactive_cacheable_worker.rb | 1 | ||||
-rw-r--r-- | app/workers/concerns/worker_attributes.rb | 22 |
6 files changed, 61 insertions, 4 deletions
diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index 7ab9a0c2a02..9c942228111 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -76,6 +76,22 @@ module ApplicationWorker get_sidekiq_options['queue'].to_s end + # Set/get which arguments can be logged and sent to Sentry. + # + # Numeric arguments are logged by default, so there is no need to + # list those. + # + # Non-numeric arguments must be listed by position, as Sidekiq + # cannot see argument names. + # + def loggable_arguments(*args) + if args.any? + @loggable_arguments = args + else + @loggable_arguments || [] + end + end + def queue_size Sidekiq::Queue.new(queue).size end @@ -84,7 +100,7 @@ module ApplicationWorker Sidekiq::Client.push_bulk('class' => self, 'args' => args_list) end - def bulk_perform_in(delay, args_list) + def bulk_perform_in(delay, args_list, batch_size: nil, batch_delay: nil) now = Time.now.to_i schedule = now + delay.to_i @@ -92,7 +108,14 @@ module ApplicationWorker raise ArgumentError, _('The schedule time must be in the future!') end - Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule) + if batch_size && batch_delay + args_list.each_slice(batch_size.to_i).with_index do |args_batch, idx| + batch_schedule = schedule + idx * batch_delay.to_i + Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch, 'at' => batch_schedule) + end + else + Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule) + end end end end diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb index 7cc23dd7c0b..05eb7fbc2cb 100644 --- a/app/workers/concerns/gitlab/github_import/queue.rb +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -15,6 +15,17 @@ module Gitlab # this is better than a project being stuck in the "import" state # forever. sidekiq_options dead: false, retry: 5 + + sidekiq_retries_exhausted do |msg, e| + Gitlab::Import::Logger.error( + event: :github_importer_exhausted, + message: msg['error_message'], + class: msg['class'], + args: msg['args'], + exception_message: e.message, + exception_backtrace: e.backtrace + ) + end end end end diff --git a/app/workers/concerns/gitlab/jira_import/import_worker.rb b/app/workers/concerns/gitlab/jira_import/import_worker.rb index 537300e6eba..fdc6e64bbaa 100644 --- a/app/workers/concerns/gitlab/jira_import/import_worker.rb +++ b/app/workers/concerns/gitlab/jira_import/import_worker.rb @@ -7,6 +7,7 @@ module Gitlab included do include ApplicationWorker + include ProjectImportOptions include Gitlab::JiraImport::QueueOptions end @@ -26,7 +27,6 @@ module Gitlab def can_import?(project) return false unless project - return false unless project.jira_issues_import_feature_flag_enabled? project.latest_jira_import&.started? end diff --git a/app/workers/concerns/project_import_options.rb b/app/workers/concerns/project_import_options.rb index c8ee5539441..4df209da29d 100644 --- a/app/workers/concerns/project_import_options.rb +++ b/app/workers/concerns/project_import_options.rb @@ -6,7 +6,7 @@ module ProjectImportOptions IMPORT_RETRY_COUNT = 5 included do - sidekiq_options retry: IMPORT_RETRY_COUNT, status_expiration: StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + sidekiq_options retry: IMPORT_RETRY_COUNT, status_expiration: Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION # We only want to mark the project as failed once we exhausted all retries sidekiq_retries_exhausted do |job| diff --git a/app/workers/concerns/reactive_cacheable_worker.rb b/app/workers/concerns/reactive_cacheable_worker.rb index e73707c2b43..189b0607605 100644 --- a/app/workers/concerns/reactive_cacheable_worker.rb +++ b/app/workers/concerns/reactive_cacheable_worker.rb @@ -7,6 +7,7 @@ module ReactiveCacheableWorker include ApplicationWorker feature_category_not_owned! + loggable_arguments 0 def self.context_for_arguments(arguments) class_name, *_other_args = arguments diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index b60179531af..b19217b15de 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -111,6 +111,28 @@ module WorkerAttributes 1 end + def tags(*values) + worker_attributes[:tags] = values + end + + def get_tags + Array(worker_attributes[:tags]) + end + + def deduplicate(strategy, options = {}) + worker_attributes[:deduplication_strategy] = strategy + worker_attributes[:deduplication_options] = options + end + + def get_deduplicate_strategy + worker_attributes[:deduplication_strategy] || + Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob::DEFAULT_STRATEGY + end + + def get_deduplication_options + worker_attributes[:deduplication_options] || {} + end + protected # Returns a worker attribute declared on this class or its parent class. |