summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--CHANGELOG.md1
-rw-r--r--app/workers/admin_email_worker.rb3
-rw-r--r--app/workers/build_coverage_worker.rb2
-rw-r--r--app/workers/build_email_worker.rb1
-rw-r--r--app/workers/build_finished_worker.rb1
-rw-r--r--app/workers/build_hooks_worker.rb2
-rw-r--r--app/workers/build_success_worker.rb2
-rw-r--r--app/workers/clear_database_cache_worker.rb1
-rw-r--r--app/workers/concerns/build_queue.rb8
-rw-r--r--app/workers/concerns/cronjob_queue.rb9
-rw-r--r--app/workers/concerns/dedicated_sidekiq_queue.rb9
-rw-r--r--app/workers/concerns/pipeline_queue.rb8
-rw-r--r--app/workers/concerns/repository_check_queue.rb8
-rw-r--r--app/workers/delete_user_worker.rb1
-rw-r--r--app/workers/email_receiver_worker.rb3
-rw-r--r--app/workers/emails_on_push_worker.rb2
-rw-r--r--app/workers/expire_build_artifacts_worker.rb1
-rw-r--r--app/workers/expire_build_instance_artifacts_worker.rb1
-rw-r--r--app/workers/git_garbage_collect_worker.rb3
-rw-r--r--app/workers/gitlab_shell_worker.rb3
-rw-r--r--app/workers/group_destroy_worker.rb3
-rw-r--r--app/workers/import_export_project_cleanup_worker.rb3
-rw-r--r--app/workers/irker_worker.rb1
-rw-r--r--app/workers/merge_worker.rb3
-rw-r--r--app/workers/new_note_worker.rb3
-rw-r--r--app/workers/pipeline_hooks_worker.rb2
-rw-r--r--app/workers/pipeline_metrics_worker.rb3
-rw-r--r--app/workers/pipeline_process_worker.rb3
-rw-r--r--app/workers/pipeline_success_worker.rb2
-rw-r--r--app/workers/pipeline_update_worker.rb3
-rw-r--r--app/workers/post_receive.rb3
-rw-r--r--app/workers/project_cache_worker.rb3
-rw-r--r--app/workers/project_destroy_worker.rb3
-rw-r--r--app/workers/project_export_worker.rb3
-rw-r--r--app/workers/project_service_worker.rb3
-rw-r--r--app/workers/project_web_hook_worker.rb3
-rw-r--r--app/workers/prune_old_events_worker.rb1
-rw-r--r--app/workers/remove_expired_group_links_worker.rb1
-rw-r--r--app/workers/remove_expired_members_worker.rb1
-rw-r--r--app/workers/repository_archive_cache_worker.rb3
-rw-r--r--app/workers/repository_check/batch_worker.rb21
-rw-r--r--app/workers/repository_check/clear_worker.rb3
-rw-r--r--app/workers/repository_check/single_repository_worker.rb3
-rw-r--r--app/workers/repository_fork_worker.rb3
-rw-r--r--app/workers/repository_import_worker.rb3
-rw-r--r--app/workers/requests_profiles_worker.rb3
-rw-r--r--app/workers/stuck_ci_builds_worker.rb1
-rw-r--r--app/workers/system_hook_worker.rb3
-rw-r--r--app/workers/trending_projects_worker.rb3
-rw-r--r--app/workers/update_merge_requests_worker.rb1
-rwxr-xr-xbin/background_jobs3
-rw-r--r--config/application.rb3
-rw-r--r--config/sidekiq_queues.yml46
-rw-r--r--db/migrate/20161019190736_migrate_sidekiq_queues_from_default.rb109
-rw-r--r--doc/development/README.md3
-rw-r--r--doc/development/sidekiq_style_guide.md38
-rw-r--r--spec/workers/concerns/build_queue_spec.rb14
-rw-r--r--spec/workers/concerns/cronjob_queue_spec.rb18
-rw-r--r--spec/workers/concerns/dedicated_sidekiq_queue_spec.rb20
-rw-r--r--spec/workers/concerns/pipeline_queue_spec.rb14
-rw-r--r--spec/workers/concerns/repository_check_queue_spec.rb18
-rw-r--r--spec/workers/every_sidekiq_worker_spec.rb44
62 files changed, 425 insertions, 68 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 518d0362d07..52d435df8f4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ Please view this file on the master branch, on stable branches it's out of date.
## 8.13.0 (2016-10-22)
- Fix save button on project pipeline settings page. (!6955)
+ - All Sidekiq workers now use their own queue
- Avoid race condition when asynchronously removing expired artifacts. (!6881)
- Improve Merge When Build Succeeds triggers and execute on pipeline success. (!6675)
- Respond with 404 Not Found for non-existent tags (Linus Thiel)
diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb
index 667fff031dd..c2dc955b27c 100644
--- a/app/workers/admin_email_worker.rb
+++ b/app/workers/admin_email_worker.rb
@@ -1,7 +1,6 @@
class AdminEmailWorker
include Sidekiq::Worker
-
- sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
+ include CronjobQueue
def perform
repository_check_failed_count = Project.where(last_repository_check_failed: true).count
diff --git a/app/workers/build_coverage_worker.rb b/app/workers/build_coverage_worker.rb
index 0680645a8db..def0ab1dde1 100644
--- a/app/workers/build_coverage_worker.rb
+++ b/app/workers/build_coverage_worker.rb
@@ -1,6 +1,6 @@
class BuildCoverageWorker
include Sidekiq::Worker
- sidekiq_options queue: :default
+ include BuildQueue
def perform(build_id)
Ci::Build.find_by(id: build_id)
diff --git a/app/workers/build_email_worker.rb b/app/workers/build_email_worker.rb
index 1c7a04a66a8..5fdb1f2baa0 100644
--- a/app/workers/build_email_worker.rb
+++ b/app/workers/build_email_worker.rb
@@ -1,5 +1,6 @@
class BuildEmailWorker
include Sidekiq::Worker
+ include BuildQueue
def perform(build_id, recipients, push_data)
recipients.each do |recipient|
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index e7286b77ac5..466410bf08c 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -1,5 +1,6 @@
class BuildFinishedWorker
include Sidekiq::Worker
+ include BuildQueue
def perform(build_id)
Ci::Build.find_by(id: build_id).try do |build|
diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb
index e22ececb3fd..9965af935d4 100644
--- a/app/workers/build_hooks_worker.rb
+++ b/app/workers/build_hooks_worker.rb
@@ -1,6 +1,6 @@
class BuildHooksWorker
include Sidekiq::Worker
- sidekiq_options queue: :default
+ include BuildQueue
def perform(build_id)
Ci::Build.find_by(id: build_id)
diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb
index 500d357ce31..e0ad5268664 100644
--- a/app/workers/build_success_worker.rb
+++ b/app/workers/build_success_worker.rb
@@ -1,6 +1,6 @@
class BuildSuccessWorker
include Sidekiq::Worker
- sidekiq_options queue: :default
+ include BuildQueue
def perform(build_id)
Ci::Build.find_by(id: build_id).try do |build|
diff --git a/app/workers/clear_database_cache_worker.rb b/app/workers/clear_database_cache_worker.rb
index c541daba50e..c4cb4733482 100644
--- a/app/workers/clear_database_cache_worker.rb
+++ b/app/workers/clear_database_cache_worker.rb
@@ -1,6 +1,7 @@
# This worker clears all cache fields in the database, working in batches.
class ClearDatabaseCacheWorker
include Sidekiq::Worker
+ include DedicatedSidekiqQueue
BATCH_SIZE = 1000
diff --git a/app/workers/concerns/build_queue.rb b/app/workers/concerns/build_queue.rb
new file mode 100644
index 00000000000..cf0ead40a8b
--- /dev/null
+++ b/app/workers/concerns/build_queue.rb
@@ -0,0 +1,8 @@
+# Concern for setting Sidekiq settings for the various CI build workers.
+module BuildQueue
+ extend ActiveSupport::Concern
+
+ included do
+ sidekiq_options queue: :build
+ end
+end
diff --git a/app/workers/concerns/cronjob_queue.rb b/app/workers/concerns/cronjob_queue.rb
new file mode 100644
index 00000000000..e918bb011e0
--- /dev/null
+++ b/app/workers/concerns/cronjob_queue.rb
@@ -0,0 +1,9 @@
+# Concern that sets various Sidekiq settings for workers executed using a
+# cronjob.
+module CronjobQueue
+ extend ActiveSupport::Concern
+
+ included do
+ sidekiq_options queue: :cronjob, retry: false
+ end
+end
diff --git a/app/workers/concerns/dedicated_sidekiq_queue.rb b/app/workers/concerns/dedicated_sidekiq_queue.rb
new file mode 100644
index 00000000000..132bae6022b
--- /dev/null
+++ b/app/workers/concerns/dedicated_sidekiq_queue.rb
@@ -0,0 +1,9 @@
+# Concern that sets the queue of a Sidekiq worker based on the worker's class
+# name/namespace.
+module DedicatedSidekiqQueue
+ extend ActiveSupport::Concern
+
+ included do
+ sidekiq_options queue: name.sub(/Worker\z/, '').underscore.tr('/', '_')
+ end
+end
diff --git a/app/workers/concerns/pipeline_queue.rb b/app/workers/concerns/pipeline_queue.rb
new file mode 100644
index 00000000000..ca3860e1d38
--- /dev/null
+++ b/app/workers/concerns/pipeline_queue.rb
@@ -0,0 +1,8 @@
+# Concern for setting Sidekiq settings for the various CI pipeline workers.
+module PipelineQueue
+ extend ActiveSupport::Concern
+
+ included do
+ sidekiq_options queue: :pipeline
+ end
+end
diff --git a/app/workers/concerns/repository_check_queue.rb b/app/workers/concerns/repository_check_queue.rb
new file mode 100644
index 00000000000..a597321ccf4
--- /dev/null
+++ b/app/workers/concerns/repository_check_queue.rb
@@ -0,0 +1,8 @@
+# Concern for setting Sidekiq settings for the various repository check workers.
+module RepositoryCheckQueue
+ extend ActiveSupport::Concern
+
+ included do
+ sidekiq_options queue: :repository_check, retry: false
+ end
+end
diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb
index 6ff361e4d80..3194c389b3d 100644
--- a/app/workers/delete_user_worker.rb
+++ b/app/workers/delete_user_worker.rb
@@ -1,5 +1,6 @@
class DeleteUserWorker
include Sidekiq::Worker
+ include DedicatedSidekiqQueue
def perform(current_user_id, delete_user_id, options = {})
delete_user = User.find(delete_user_id)
diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb
index 842eebdea9e..d3f7e479a8d 100644
--- a/app/workers/email_receiver_worker.rb
+++ b/app/workers/email_receiver_worker.rb
@@ -1,7 +1,6 @@
class EmailReceiverWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :incoming_email
+ include DedicatedSidekiqQueue
def perform(raw)
return unless Gitlab::IncomingEmail.enabled?
diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb
index 1dc7e0adef7..b9cd49985dc 100644
--- a/app/workers/emails_on_push_worker.rb
+++ b/app/workers/emails_on_push_worker.rb
@@ -1,7 +1,7 @@
class EmailsOnPushWorker
include Sidekiq::Worker
+ include DedicatedSidekiqQueue
- sidekiq_options queue: :mailers
attr_reader :email, :skip_premailer
def perform(project_id, recipients, push_data, options = {})
diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb
index 174eabff9fd..a27585fd389 100644
--- a/app/workers/expire_build_artifacts_worker.rb
+++ b/app/workers/expire_build_artifacts_worker.rb
@@ -1,5 +1,6 @@
class ExpireBuildArtifactsWorker
include Sidekiq::Worker
+ include CronjobQueue
def perform
Rails.logger.info 'Scheduling removal of build artifacts'
diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb
index d9e2cc37bb3..eb403c134d1 100644
--- a/app/workers/expire_build_instance_artifacts_worker.rb
+++ b/app/workers/expire_build_instance_artifacts_worker.rb
@@ -1,5 +1,6 @@
class ExpireBuildInstanceArtifactsWorker
include Sidekiq::Worker
+ include DedicatedSidekiqQueue
def perform(build_id)
build = Ci::Build
diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index a6cefd4d601..65f8093b5b0 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -1,8 +1,9 @@
class GitGarbageCollectWorker
include Sidekiq::Worker
include Gitlab::ShellAdapter
+ include DedicatedSidekiqQueue
- sidekiq_options queue: :gitlab_shell, retry: false
+ sidekiq_options retry: false
def perform(project_id)
project = Project.find(project_id)
diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb
index cfeda88bbc5..964287a1793 100644
--- a/app/workers/gitlab_shell_worker.rb
+++ b/app/workers/gitlab_shell_worker.rb
@@ -1,8 +1,7 @@
class GitlabShellWorker
include Sidekiq::Worker
include Gitlab::ShellAdapter
-
- sidekiq_options queue: :gitlab_shell
+ include DedicatedSidekiqQueue
def perform(action, *arg)
gitlab_shell.send(action, *arg)
diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb
index 5048746f09b..a49a5fd0855 100644
--- a/app/workers/group_destroy_worker.rb
+++ b/app/workers/group_destroy_worker.rb
@@ -1,7 +1,6 @@
class GroupDestroyWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include DedicatedSidekiqQueue
def perform(group_id, user_id)
begin
diff --git a/app/workers/import_export_project_cleanup_worker.rb b/app/workers/import_export_project_cleanup_worker.rb
index 72e3a9ae734..7957ed807ab 100644
--- a/app/workers/import_export_project_cleanup_worker.rb
+++ b/app/workers/import_export_project_cleanup_worker.rb
@@ -1,7 +1,6 @@
class ImportExportProjectCleanupWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include CronjobQueue
def perform
ImportExportCleanUpService.new.execute
diff --git a/app/workers/irker_worker.rb b/app/workers/irker_worker.rb
index 19f38358eb5..7e44b241743 100644
--- a/app/workers/irker_worker.rb
+++ b/app/workers/irker_worker.rb
@@ -3,6 +3,7 @@ require 'socket'
class IrkerWorker
include Sidekiq::Worker
+ include DedicatedSidekiqQueue
def perform(project_id, chans, colors, push_data, settings)
project = Project.find(project_id)
diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb
index c87c0a252b1..79efca4f2f9 100644
--- a/app/workers/merge_worker.rb
+++ b/app/workers/merge_worker.rb
@@ -1,7 +1,6 @@
class MergeWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include DedicatedSidekiqQueue
def perform(merge_request_id, current_user_id, params)
params = params.with_indifferent_access
diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb
index 1b3232cd365..c3e62bb88c0 100644
--- a/app/workers/new_note_worker.rb
+++ b/app/workers/new_note_worker.rb
@@ -1,7 +1,6 @@
class NewNoteWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include DedicatedSidekiqQueue
def perform(note_id, note_params)
note = Note.find(note_id)
diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb
index ab5e9f6daad..7e36eacebf8 100644
--- a/app/workers/pipeline_hooks_worker.rb
+++ b/app/workers/pipeline_hooks_worker.rb
@@ -1,6 +1,6 @@
class PipelineHooksWorker
include Sidekiq::Worker
- sidekiq_options queue: :default
+ include PipelineQueue
def perform(pipeline_id)
Ci::Pipeline.find_by(id: pipeline_id)
diff --git a/app/workers/pipeline_metrics_worker.rb b/app/workers/pipeline_metrics_worker.rb
index 7bb92df3bbd..34f6ef161fb 100644
--- a/app/workers/pipeline_metrics_worker.rb
+++ b/app/workers/pipeline_metrics_worker.rb
@@ -1,7 +1,6 @@
class PipelineMetricsWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include PipelineQueue
def perform(pipeline_id)
Ci::Pipeline.find_by(id: pipeline_id).try do |pipeline|
diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb
index f44227d7086..357e4a9a1c3 100644
--- a/app/workers/pipeline_process_worker.rb
+++ b/app/workers/pipeline_process_worker.rb
@@ -1,7 +1,6 @@
class PipelineProcessWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include PipelineQueue
def perform(pipeline_id)
Ci::Pipeline.find_by(id: pipeline_id)
diff --git a/app/workers/pipeline_success_worker.rb b/app/workers/pipeline_success_worker.rb
index 5dd443fea59..2aa6fff24da 100644
--- a/app/workers/pipeline_success_worker.rb
+++ b/app/workers/pipeline_success_worker.rb
@@ -1,6 +1,6 @@
class PipelineSuccessWorker
include Sidekiq::Worker
- sidekiq_options queue: :default
+ include PipelineQueue
def perform(pipeline_id)
Ci::Pipeline.find_by(id: pipeline_id).try do |pipeline|
diff --git a/app/workers/pipeline_update_worker.rb b/app/workers/pipeline_update_worker.rb
index 44a7f24e401..96c4152c674 100644
--- a/app/workers/pipeline_update_worker.rb
+++ b/app/workers/pipeline_update_worker.rb
@@ -1,7 +1,6 @@
class PipelineUpdateWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include PipelineQueue
def perform(pipeline_id)
Ci::Pipeline.find_by(id: pipeline_id)
diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb
index a9a2b716005..eee0ca12af9 100644
--- a/app/workers/post_receive.rb
+++ b/app/workers/post_receive.rb
@@ -1,7 +1,6 @@
class PostReceive
include Sidekiq::Worker
-
- sidekiq_options queue: :post_receive
+ include DedicatedSidekiqQueue
def perform(repo_path, identifier, changes)
if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb
index 0d524e88dc3..71b274e0c99 100644
--- a/app/workers/project_cache_worker.rb
+++ b/app/workers/project_cache_worker.rb
@@ -5,8 +5,7 @@
# storage engine as much.
class ProjectCacheWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include DedicatedSidekiqQueue
LEASE_TIMEOUT = 15.minutes.to_i
diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb
index 3062301a9b1..b462327490e 100644
--- a/app/workers/project_destroy_worker.rb
+++ b/app/workers/project_destroy_worker.rb
@@ -1,7 +1,6 @@
class ProjectDestroyWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include DedicatedSidekiqQueue
def perform(project_id, user_id, params)
begin
diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb
index 615311e63f5..6009aa1b191 100644
--- a/app/workers/project_export_worker.rb
+++ b/app/workers/project_export_worker.rb
@@ -1,7 +1,8 @@
class ProjectExportWorker
include Sidekiq::Worker
+ include DedicatedSidekiqQueue
- sidekiq_options queue: :gitlab_shell, retry: 3
+ sidekiq_options retry: 3
def perform(current_user_id, project_id)
current_user = User.find(current_user_id)
diff --git a/app/workers/project_service_worker.rb b/app/workers/project_service_worker.rb
index 64d39c4d3f7..fdfdeab7b41 100644
--- a/app/workers/project_service_worker.rb
+++ b/app/workers/project_service_worker.rb
@@ -1,7 +1,6 @@
class ProjectServiceWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :project_web_hook
+ include DedicatedSidekiqQueue
def perform(hook_id, data)
data = data.with_indifferent_access
diff --git a/app/workers/project_web_hook_worker.rb b/app/workers/project_web_hook_worker.rb
index fb878965288..efb85eafd15 100644
--- a/app/workers/project_web_hook_worker.rb
+++ b/app/workers/project_web_hook_worker.rb
@@ -1,7 +1,6 @@
class ProjectWebHookWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :project_web_hook
+ include DedicatedSidekiqQueue
def perform(hook_id, data, hook_name)
data = data.with_indifferent_access
diff --git a/app/workers/prune_old_events_worker.rb b/app/workers/prune_old_events_worker.rb
index 5883cafe1d1..392abb9c21b 100644
--- a/app/workers/prune_old_events_worker.rb
+++ b/app/workers/prune_old_events_worker.rb
@@ -1,5 +1,6 @@
class PruneOldEventsWorker
include Sidekiq::Worker
+ include CronjobQueue
def perform
# Contribution calendar shows maximum 12 months of events.
diff --git a/app/workers/remove_expired_group_links_worker.rb b/app/workers/remove_expired_group_links_worker.rb
index 246c8b6650a..2a619f83410 100644
--- a/app/workers/remove_expired_group_links_worker.rb
+++ b/app/workers/remove_expired_group_links_worker.rb
@@ -1,5 +1,6 @@
class RemoveExpiredGroupLinksWorker
include Sidekiq::Worker
+ include CronjobQueue
def perform
ProjectGroupLink.expired.destroy_all
diff --git a/app/workers/remove_expired_members_worker.rb b/app/workers/remove_expired_members_worker.rb
index cf765af97ce..31f652e5f9b 100644
--- a/app/workers/remove_expired_members_worker.rb
+++ b/app/workers/remove_expired_members_worker.rb
@@ -1,5 +1,6 @@
class RemoveExpiredMembersWorker
include Sidekiq::Worker
+ include CronjobQueue
def perform
Member.expired.find_each do |member|
diff --git a/app/workers/repository_archive_cache_worker.rb b/app/workers/repository_archive_cache_worker.rb
index a2e49c61f59..e47069df189 100644
--- a/app/workers/repository_archive_cache_worker.rb
+++ b/app/workers/repository_archive_cache_worker.rb
@@ -1,7 +1,6 @@
class RepositoryArchiveCacheWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include CronjobQueue
def perform
RepositoryArchiveCleanUpService.new.execute
diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb
index a3e16fa5212..c3e7491ec4e 100644
--- a/app/workers/repository_check/batch_worker.rb
+++ b/app/workers/repository_check/batch_worker.rb
@@ -1,14 +1,13 @@
module RepositoryCheck
class BatchWorker
include Sidekiq::Worker
-
+ include CronjobQueue
+
RUN_TIME = 3600
-
- sidekiq_options retry: false
-
+
def perform
start = Time.now
-
+
# This loop will break after a little more than one hour ('a little
# more' because `git fsck` may take a few minutes), or if it runs out of
# projects to check. By default sidekiq-cron will start a new
@@ -17,15 +16,15 @@ module RepositoryCheck
project_ids.each do |project_id|
break if Time.now - start >= RUN_TIME
break unless current_settings.repository_checks_enabled
-
+
next unless try_obtain_lease(project_id)
-
+
SingleRepositoryWorker.new.perform(project_id)
end
end
-
+
private
-
+
# Project.find_each does not support WHERE clauses and
# Project.find_in_batches does not support ordering. So we just build an
# array of ID's. This is OK because we do it only once an hour, because
@@ -39,7 +38,7 @@ module RepositoryCheck
reorder('last_repository_check_at ASC').limit(limit).pluck(:id)
never_checked_projects + old_check_projects
end
-
+
def try_obtain_lease(id)
# Use a 24-hour timeout because on servers/projects where 'git fsck' is
# super slow we definitely do not want to run it twice in parallel.
@@ -48,7 +47,7 @@ module RepositoryCheck
timeout: 24.hours
).try_obtain
end
-
+
def current_settings
# No caching of the settings! If we cache them and an admin disables
# this feature, an active RepositoryCheckWorker would keep going for up
diff --git a/app/workers/repository_check/clear_worker.rb b/app/workers/repository_check/clear_worker.rb
index b7202ddff34..1f1b38540ee 100644
--- a/app/workers/repository_check/clear_worker.rb
+++ b/app/workers/repository_check/clear_worker.rb
@@ -1,8 +1,7 @@
module RepositoryCheck
class ClearWorker
include Sidekiq::Worker
-
- sidekiq_options retry: false
+ include RepositoryCheckQueue
def perform
# Do small batched updates because these updates will be slow and locking
diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb
index 98ddf5d0688..3d8bfc6fc6c 100644
--- a/app/workers/repository_check/single_repository_worker.rb
+++ b/app/workers/repository_check/single_repository_worker.rb
@@ -1,8 +1,7 @@
module RepositoryCheck
class SingleRepositoryWorker
include Sidekiq::Worker
-
- sidekiq_options retry: false
+ include RepositoryCheckQueue
def perform(project_id)
project = Project.find(project_id)
diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb
index 61ed1c38ac4..efc99ec962a 100644
--- a/app/workers/repository_fork_worker.rb
+++ b/app/workers/repository_fork_worker.rb
@@ -1,8 +1,7 @@
class RepositoryForkWorker
include Sidekiq::Worker
include Gitlab::ShellAdapter
-
- sidekiq_options queue: :gitlab_shell
+ include DedicatedSidekiqQueue
def perform(project_id, forked_from_repository_storage_path, source_path, target_path)
Gitlab::Metrics.add_event(:fork_repository,
diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb
index d2ca8813ab9..c8a77e21c12 100644
--- a/app/workers/repository_import_worker.rb
+++ b/app/workers/repository_import_worker.rb
@@ -1,8 +1,7 @@
class RepositoryImportWorker
include Sidekiq::Worker
include Gitlab::ShellAdapter
-
- sidekiq_options queue: :gitlab_shell
+ include DedicatedSidekiqQueue
attr_accessor :project, :current_user
diff --git a/app/workers/requests_profiles_worker.rb b/app/workers/requests_profiles_worker.rb
index 9dd228a2483..703b025d76e 100644
--- a/app/workers/requests_profiles_worker.rb
+++ b/app/workers/requests_profiles_worker.rb
@@ -1,7 +1,6 @@
class RequestsProfilesWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :default
+ include CronjobQueue
def perform
Gitlab::RequestProfiler.remove_all_profiles
diff --git a/app/workers/stuck_ci_builds_worker.rb b/app/workers/stuck_ci_builds_worker.rb
index 6828013b377..b70df5a1afa 100644
--- a/app/workers/stuck_ci_builds_worker.rb
+++ b/app/workers/stuck_ci_builds_worker.rb
@@ -1,5 +1,6 @@
class StuckCiBuildsWorker
include Sidekiq::Worker
+ include CronjobQueue
BUILD_STUCK_TIMEOUT = 1.day
diff --git a/app/workers/system_hook_worker.rb b/app/workers/system_hook_worker.rb
index a122c274763..baf2f12eeac 100644
--- a/app/workers/system_hook_worker.rb
+++ b/app/workers/system_hook_worker.rb
@@ -1,7 +1,6 @@
class SystemHookWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :system_hook
+ include DedicatedSidekiqQueue
def perform(hook_id, data, hook_name)
SystemHook.find(hook_id).execute(data, hook_name)
diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb
index df4c4a6628b..0531630d13a 100644
--- a/app/workers/trending_projects_worker.rb
+++ b/app/workers/trending_projects_worker.rb
@@ -1,7 +1,6 @@
class TrendingProjectsWorker
include Sidekiq::Worker
-
- sidekiq_options queue: :trending_projects
+ include CronjobQueue
def perform
Rails.logger.info('Refreshing trending projects')
diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb
index 03f0528cdae..acc4d858136 100644
--- a/app/workers/update_merge_requests_worker.rb
+++ b/app/workers/update_merge_requests_worker.rb
@@ -1,5 +1,6 @@
class UpdateMergeRequestsWorker
include Sidekiq::Worker
+ include DedicatedSidekiqQueue
def perform(project_id, user_id, oldrev, newrev, ref)
project = Project.find_by(id: project_id)
diff --git a/bin/background_jobs b/bin/background_jobs
index 25a578a1c49..f28e2f722dc 100755
--- a/bin/background_jobs
+++ b/bin/background_jobs
@@ -4,6 +4,7 @@ cd $(dirname $0)/..
app_root=$(pwd)
sidekiq_pidfile="$app_root/tmp/pids/sidekiq.pid"
sidekiq_logfile="$app_root/log/sidekiq.log"
+sidekiq_config="$app_root/config/sidekiq_queues.yml"
gitlab_user=$(ls -l config.ru | awk '{print $3}')
warn()
@@ -37,7 +38,7 @@ start_no_deamonize()
start_sidekiq()
{
- exec bundle exec sidekiq -q post_receive -q mailers -q archive_repo -q system_hook -q project_web_hook -q gitlab_shell -q incoming_email -q runner -q common -q default -e $RAILS_ENV -P $sidekiq_pidfile "$@"
+ exec bundle exec sidekiq -C "${sidekiq_config}" -e $RAILS_ENV -P $sidekiq_pidfile "$@"
}
load_ok()
diff --git a/config/application.rb b/config/application.rb
index f3337b00dc6..92c8467e7f4 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -24,7 +24,8 @@ module Gitlab
#{config.root}/app/models/ci
#{config.root}/app/models/hooks
#{config.root}/app/models/members
- #{config.root}/app/models/project_services))
+ #{config.root}/app/models/project_services
+ #{config.root}/app/workers/concerns))
config.generators.templates.push("#{config.root}/generator_templates")
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
new file mode 100644
index 00000000000..c2e880e891f
--- /dev/null
+++ b/config/sidekiq_queues.yml
@@ -0,0 +1,46 @@
+# This configuration file should be exclusively used to set queue settings for
+# Sidekiq. Any other setting should be specified using the Sidekiq CLI or the
+# Sidekiq Ruby API (see config/initializers/sidekiq.rb).
+---
+# All the queues to process and their weights. Every queue _must_ have a weight
+# defined.
+#
+# The available weights are as follows
+#
+# 1: low priority
+# 2: medium priority
+# 3: high priority
+# 5: _super_ high priority, this should only be used for _very_ important queues
+#
+# As per http://stackoverflow.com/a/21241357/290102 the formula for calculating
+# the likelihood of a job being popped off a queue (given all queues have work
+# to perform) is:
+#
+# chance = (queue weight / total weight of all queues) * 100
+:queues:
+ - [post_receive, 5]
+ - [merge, 5]
+ - [update_merge_requests, 3]
+ - [new_note, 2]
+ - [build, 2]
+ - [pipeline, 2]
+ - [gitlab_shell, 2]
+ - [email_receiver, 2]
+ - [emails_on_push, 2]
+ - [repository_fork, 1]
+ - [repository_import, 1]
+ - [project_service, 1]
+ - [clear_database_cache, 1]
+ - [delete_user, 1]
+ - [expire_build_instance_artifacts, 1]
+ - [group_destroy, 1]
+ - [irker, 1]
+ - [project_cache, 1]
+ - [project_destroy, 1]
+ - [project_export, 1]
+ - [project_web_hook, 1]
+ - [repository_check, 1]
+ - [system_hook, 1]
+ - [git_garbage_collect, 1]
+ - [cronjob, 1]
+ - [default, 1]
diff --git a/db/migrate/20161019190736_migrate_sidekiq_queues_from_default.rb b/db/migrate/20161019190736_migrate_sidekiq_queues_from_default.rb
new file mode 100644
index 00000000000..e875213ab96
--- /dev/null
+++ b/db/migrate/20161019190736_migrate_sidekiq_queues_from_default.rb
@@ -0,0 +1,109 @@
+require 'json'
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MigrateSidekiqQueuesFromDefault < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = true
+
+ DOWNTIME_REASON = <<-EOF
+ Moving Sidekiq jobs from queues requires Sidekiq to be stopped. Not stopping
+ Sidekiq will result in the loss of jobs that are scheduled after this
+ migration completes.
+ EOF
+
+ disable_ddl_transaction!
+
+ # Jobs for which the queue names have been changed (e.g. multiple workers
+ # using the same non-default queue).
+ #
+ # The keys are the old queue names, the values the jobs to move and their new
+ # queue names.
+ RENAMED_QUEUES = {
+ gitlab_shell: {
+ 'GitGarbageCollectorWorker' => :git_garbage_collector,
+ 'ProjectExportWorker' => :project_export,
+ 'RepositoryForkWorker' => :repository_fork,
+ 'RepositoryImportWorker' => :repository_import
+ },
+ project_web_hook: {
+ 'ProjectServiceWorker' => :project_service
+ },
+ incoming_email: {
+ 'EmailReceiverWorker' => :email_receiver
+ },
+ mailers: {
+ 'EmailsOnPushWorker' => :emails_on_push
+ },
+ default: {
+ 'AdminEmailWorker' => :cronjob,
+ 'BuildCoverageWorker' => :build,
+ 'BuildEmailWorker' => :build,
+ 'BuildFinishedWorker' => :build,
+ 'BuildHooksWorker' => :build,
+ 'BuildSuccessWorker' => :build,
+ 'ClearDatabaseCacheWorker' => :clear_database_cache,
+ 'DeleteUserWorker' => :delete_user,
+ 'ExpireBuildArtifactsWorker' => :cronjob,
+ 'ExpireBuildInstanceArtifactsWorker' => :expire_build_instance_artifacts,
+ 'GroupDestroyWorker' => :group_destroy,
+ 'ImportExportProjectCleanupWorker' => :cronjob,
+ 'IrkerWorker' => :irker,
+ 'MergeWorker' => :merge,
+ 'NewNoteWorker' => :new_note,
+ 'PipelineHooksWorker' => :pipeline,
+ 'PipelineMetricsWorker' => :pipeline,
+ 'PipelineProcessWorker' => :pipeline,
+ 'PipelineSuccessWorker' => :pipeline,
+ 'PipelineUpdateWorker' => :pipeline,
+ 'ProjectCacheWorker' => :project_cache,
+ 'ProjectDestroyWorker' => :project_destroy,
+ 'PruneOldEventsWorker' => :cronjob,
+ 'RemoveExpiredGroupLinksWorker' => :cronjob,
+ 'RemoveExpiredMembersWorker' => :cronjob,
+ 'RepositoryArchiveCacheWorker' => :cronjob,
+ 'RepositoryCheck::BatchWorker' => :cronjob,
+ 'RepositoryCheck::ClearWorker' => :repository_check,
+ 'RepositoryCheck::SingleRepositoryWorker' => :repository_check,
+ 'RequestsProfilesWorker' => :cronjob,
+ 'StuckCiBuildsWorker' => :cronjob,
+ 'UpdateMergeRequestsWorker' => :update_merge_requests
+ }
+ }
+
+ def up
+ Sidekiq.redis do |redis|
+ RENAMED_QUEUES.each do |queue, jobs|
+ migrate_from_queue(redis, queue, jobs)
+ end
+ end
+ end
+
+ def down
+ Sidekiq.redis do |redis|
+ RENAMED_QUEUES.each do |dest_queue, jobs|
+ jobs.each do |worker, from_queue|
+ migrate_from_queue(redis, from_queue, worker => dest_queue)
+ end
+ end
+ end
+ end
+
+ def migrate_from_queue(redis, queue, job_mapping)
+ while job = redis.lpop("queue:#{queue}")
+ payload = JSON.load(job)
+ new_queue = job_mapping[payload['class']]
+
+ # If we have no target queue to migrate to we're probably dealing with
+ # some ancient job for which the worker no longer exists. In that case
+ # there's no sane option we can take, other than just dropping the job.
+ next unless new_queue
+
+ payload['queue'] = new_queue
+
+ redis.lpush("queue:#{new_queue}", JSON.dump(payload))
+ end
+ end
+end
diff --git a/doc/development/README.md b/doc/development/README.md
index 9706cb1de7f..fb6a8a5b095 100644
--- a/doc/development/README.md
+++ b/doc/development/README.md
@@ -14,7 +14,8 @@
- [Testing standards and style guidelines](testing.md)
- [UI guide](ui_guide.md) for building GitLab with existing CSS styles and elements
- [Frontend guidelines](frontend.md)
-- [SQL guidelines](sql.md) for SQL guidelines
+- [SQL guidelines](sql.md) for working with SQL queries
+- [Sidekiq guidelines](sidekiq_style_guide.md) for working with Sidekiq workers
## Process
diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md
new file mode 100644
index 00000000000..e3a20f29a09
--- /dev/null
+++ b/doc/development/sidekiq_style_guide.md
@@ -0,0 +1,38 @@
+# Sidekiq Style Guide
+
+This document outlines various guidelines that should be followed when adding or
+modifying Sidekiq workers.
+
+## Default Queue
+
+Use of the "default" queue is not allowed. Every worker should use a queue that
+matches the worker's purpose the closest. For example, workers that are to be
+executed periodically should use the "cronjob" queue.
+
+A list of all available queues can be found in `config/sidekiq_queues.yml`.
+
+## Dedicated Queues
+
+Most workers should use their own queue. To ease this process a worker can
+include the `DedicatedSidekiqQueue` concern as follows:
+
+```ruby
+class ProcessSomethingWorker
+ include Sidekiq::Worker
+ include DedicatedSidekiqQueue
+end
+```
+
+This will set the queue name based on the class' name, minus the `Worker`
+suffix. In the above example this would lead to the queue being
+`process_something`.
+
+In some cases multiple workers do use the same queue. For example, the various
+workers for updating CI pipelines all use the `pipeline` queue. Adding workers
+to existing queues should be done with care, as adding more workers can lead to
+slow jobs blocking work (even for different jobs) on the shared queue.
+
+## Tests
+
+Each Sidekiq worker must be tested using RSpec, just like any other class. These
+tests should be placed in `spec/workers`.
diff --git a/spec/workers/concerns/build_queue_spec.rb b/spec/workers/concerns/build_queue_spec.rb
new file mode 100644
index 00000000000..6bf955e0be2
--- /dev/null
+++ b/spec/workers/concerns/build_queue_spec.rb
@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe BuildQueue do
+ let(:worker) do
+ Class.new do
+ include Sidekiq::Worker
+ include BuildQueue
+ end
+ end
+
+ it 'sets the queue name of a worker' do
+ expect(worker.sidekiq_options['queue'].to_s).to eq('build')
+ end
+end
diff --git a/spec/workers/concerns/cronjob_queue_spec.rb b/spec/workers/concerns/cronjob_queue_spec.rb
new file mode 100644
index 00000000000..5d1336c21a6
--- /dev/null
+++ b/spec/workers/concerns/cronjob_queue_spec.rb
@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe CronjobQueue do
+ let(:worker) do
+ Class.new do
+ include Sidekiq::Worker
+ include CronjobQueue
+ end
+ end
+
+ it 'sets the queue name of a worker' do
+ expect(worker.sidekiq_options['queue'].to_s).to eq('cronjob')
+ end
+
+ it 'disables retrying of failed jobs' do
+ expect(worker.sidekiq_options['retry']).to eq(false)
+ end
+end
diff --git a/spec/workers/concerns/dedicated_sidekiq_queue_spec.rb b/spec/workers/concerns/dedicated_sidekiq_queue_spec.rb
new file mode 100644
index 00000000000..512baec8b7e
--- /dev/null
+++ b/spec/workers/concerns/dedicated_sidekiq_queue_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe DedicatedSidekiqQueue do
+ let(:worker) do
+ Class.new do
+ def self.name
+ 'Foo::Bar::DummyWorker'
+ end
+
+ include Sidekiq::Worker
+ include DedicatedSidekiqQueue
+ end
+ end
+
+ describe 'queue names' do
+ it 'sets the queue name based on the class name' do
+ expect(worker.sidekiq_options['queue']).to eq('foo_bar_dummy')
+ end
+ end
+end
diff --git a/spec/workers/concerns/pipeline_queue_spec.rb b/spec/workers/concerns/pipeline_queue_spec.rb
new file mode 100644
index 00000000000..40794d0e42a
--- /dev/null
+++ b/spec/workers/concerns/pipeline_queue_spec.rb
@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe PipelineQueue do
+ let(:worker) do
+ Class.new do
+ include Sidekiq::Worker
+ include PipelineQueue
+ end
+ end
+
+ it 'sets the queue name of a worker' do
+ expect(worker.sidekiq_options['queue'].to_s).to eq('pipeline')
+ end
+end
diff --git a/spec/workers/concerns/repository_check_queue_spec.rb b/spec/workers/concerns/repository_check_queue_spec.rb
new file mode 100644
index 00000000000..8868e969829
--- /dev/null
+++ b/spec/workers/concerns/repository_check_queue_spec.rb
@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe RepositoryCheckQueue do
+ let(:worker) do
+ Class.new do
+ include Sidekiq::Worker
+ include RepositoryCheckQueue
+ end
+ end
+
+ it 'sets the queue name of a worker' do
+ expect(worker.sidekiq_options['queue'].to_s).to eq('repository_check')
+ end
+
+ it 'disables retrying of failed jobs' do
+ expect(worker.sidekiq_options['retry']).to eq(false)
+ end
+end
diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb
new file mode 100644
index 00000000000..fc9adf47c1e
--- /dev/null
+++ b/spec/workers/every_sidekiq_worker_spec.rb
@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe 'Every Sidekiq worker' do
+ let(:workers) do
+ root = Rails.root.join('app', 'workers')
+ concerns = root.join('concerns').to_s
+
+ workers = Dir[root.join('**', '*.rb')].
+ reject { |path| path.start_with?(concerns) }
+
+ workers.map do |path|
+ ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
+
+ ns.camelize.constantize
+ end
+ end
+
+ it 'does not use the default queue' do
+ workers.each do |worker|
+ expect(worker.sidekiq_options['queue'].to_s).not_to eq('default')
+ end
+ end
+
+ it 'uses the cronjob queue when the worker runs as a cronjob' do
+ cron_workers = Settings.cron_jobs.
+ map { |job_name, options| options['job_class'].constantize }.
+ to_set
+
+ workers.each do |worker|
+ next unless cron_workers.include?(worker)
+
+ expect(worker.sidekiq_options['queue'].to_s).to eq('cronjob')
+ end
+ end
+
+ it 'defines the queue in the Sidekiq configuration file' do
+ config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
+ queue_names = config[:queues].map { |(queue, _)| queue }.to_set
+
+ workers.each do |worker|
+ expect(queue_names).to include(worker.sidekiq_options['queue'].to_s)
+ end
+ end
+end