diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 09:08:42 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 09:08:42 +0000 |
| commit | b76ae638462ab0f673e5915986070518dd3f9ad3 (patch) | |
| tree | bdab0533383b52873be0ec0eb4d3c66598ff8b91 /app/workers | |
| parent | 434373eabe7b4be9593d18a585fb763f1e5f1a6f (diff) | |
| download | gitlab-ce-b76ae638462ab0f673e5915986070518dd3f9ad3.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42
Diffstat (limited to 'app/workers')
288 files changed, 813 insertions, 182 deletions
diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb index ea7709c649f..0877b11210d 100644 --- a/app/workers/admin_email_worker.rb +++ b/app/workers/admin_email_worker.rb @@ -3,7 +3,8 @@ class AdminEmailWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 8d08beb56aa..f326ae0dec8 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -247,6 +247,15 @@ :idempotent: true :tags: - :exclude_from_kubernetes +- :name: cronjob:database_drop_detached_partitions + :worker_name: Database::DropDetachedPartitionsWorker + :feature_category: :database + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:database_partition_management :worker_name: Database::PartitionManagementWorker :feature_category: :database @@ -256,12 +265,21 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:environments_auto_delete_cron + :worker_name: Environments::AutoDeleteCronWorker + :feature_category: :continuous_delivery + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:environments_auto_stop_cron :worker_name: Environments::AutoStopCronWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: [] @@ -279,7 +297,7 @@ :feature_category: :service_ping :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: [] @@ -381,7 +399,7 @@ :feature_category: :pages :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: [] @@ -390,7 +408,7 @@ :feature_category: :pages :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: [] @@ -903,7 +921,7 @@ :feature_category: :importers :has_external_dependencies: true :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: @@ -913,7 +931,7 @@ :feature_category: :importers :has_external_dependencies: true :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: @@ -1020,6 +1038,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: hashed_storage:hashed_storage_project_migrate :worker_name: HashedStorage::ProjectMigrateWorker :feature_category: :source_code_management @@ -1030,6 +1049,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: hashed_storage:hashed_storage_project_rollback :worker_name: HashedStorage::ProjectRollbackWorker :feature_category: :source_code_management @@ -1040,6 +1060,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: hashed_storage:hashed_storage_rollbacker :worker_name: HashedStorage::RollbackerWorker :feature_category: :source_code_management @@ -1050,6 +1071,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: incident_management:clusters_applications_check_prometheus_health :worker_name: Clusters::Applications::CheckPrometheusHealthWorker :feature_category: :incident_management @@ -1064,7 +1086,7 @@ :feature_category: :incident_management :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 2 :idempotent: :tags: @@ -1083,7 +1105,7 @@ :feature_category: :incident_management :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 2 :idempotent: true :tags: [] @@ -1096,6 +1118,15 @@ :weight: 1 :idempotent: :tags: [] +- :name: jira_connect:jira_connect_retry_request + :worker_name: JiraConnect::RetryRequestWorker + :feature_category: :integrations + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: jira_connect:jira_connect_sync_branch :worker_name: JiraConnect::SyncBranchWorker :feature_category: :integrations @@ -2028,7 +2059,8 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :needs_own_queue - :name: emails_on_push :worker_name: EmailsOnPushWorker :feature_category: :source_code_management @@ -2127,7 +2159,7 @@ :feature_category: :metrics :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: true :tags: @@ -2369,7 +2401,7 @@ :feature_category: :product_analytics :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: true :tags: @@ -2426,7 +2458,7 @@ :feature_category: :pages :has_external_dependencies: :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: @@ -2771,7 +2803,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :needs_own_queue - :name: snippets_schedule_bulk_repository_shard_moves :worker_name: Snippets::ScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly diff --git a/app/workers/analytics/usage_trends/count_job_trigger_worker.rb b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb index f2d4404a964..41ef75ac20a 100644 --- a/app/workers/analytics/usage_trends/count_job_trigger_worker.rb +++ b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb @@ -6,7 +6,8 @@ module Analytics extend ::Gitlab::Utils::Override include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext DEFAULT_DELAY = 3.minutes.freeze diff --git a/app/workers/analytics/usage_trends/counter_job_worker.rb b/app/workers/analytics/usage_trends/counter_job_worker.rb index f4dc497d25f..bfb8a435939 100644 --- a/app/workers/analytics/usage_trends/counter_job_worker.rb +++ b/app/workers/analytics/usage_trends/counter_job_worker.rb @@ -6,6 +6,8 @@ module Analytics extend ::Gitlab::Utils::Override include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :devops_reports diff --git a/app/workers/approve_blocked_pending_approval_users_worker.rb b/app/workers/approve_blocked_pending_approval_users_worker.rb index ff72aaad3ce..fdf1bd99558 100644 --- a/app/workers/approve_blocked_pending_approval_users_worker.rb +++ b/app/workers/approve_blocked_pending_approval_users_worker.rb @@ -3,6 +3,8 @@ class ApproveBlockedPendingApprovalUsersWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 idempotent! diff --git a/app/workers/authorized_keys_worker.rb b/app/workers/authorized_keys_worker.rb index 953f493ea2c..039fe629a61 100644 --- a/app/workers/authorized_keys_worker.rb +++ b/app/workers/authorized_keys_worker.rb @@ -3,6 +3,8 @@ class AuthorizedKeysWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 PERMITTED_ACTIONS = %w[add_key remove_key].freeze diff --git a/app/workers/authorized_project_update/periodic_recalculate_worker.rb b/app/workers/authorized_project_update/periodic_recalculate_worker.rb index 2f6a9c42c0c..7b31224d151 100644 --- a/app/workers/authorized_project_update/periodic_recalculate_worker.rb +++ b/app/workers/authorized_project_update/periodic_recalculate_worker.rb @@ -4,7 +4,8 @@ module AuthorizedProjectUpdate class PeriodicRecalculateWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # This worker does not perform work scoped to a context include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/authorized_project_update/project_create_worker.rb b/app/workers/authorized_project_update/project_create_worker.rb index 52b740b4efe..1f19168cd36 100644 --- a/app/workers/authorized_project_update/project_create_worker.rb +++ b/app/workers/authorized_project_update/project_create_worker.rb @@ -4,6 +4,8 @@ module AuthorizedProjectUpdate class ProjectCreateWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :authentication_and_authorization diff --git a/app/workers/authorized_project_update/project_group_link_create_worker.rb b/app/workers/authorized_project_update/project_group_link_create_worker.rb index d887a2ce25f..d83981c4ce1 100644 --- a/app/workers/authorized_project_update/project_group_link_create_worker.rb +++ b/app/workers/authorized_project_update/project_group_link_create_worker.rb @@ -4,6 +4,8 @@ module AuthorizedProjectUpdate class ProjectGroupLinkCreateWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :authentication_and_authorization diff --git a/app/workers/authorized_project_update/project_recalculate_worker.rb b/app/workers/authorized_project_update/project_recalculate_worker.rb index 3f0672992ef..4d350d95e7e 100644 --- a/app/workers/authorized_project_update/project_recalculate_worker.rb +++ b/app/workers/authorized_project_update/project_recalculate_worker.rb @@ -3,6 +3,8 @@ module AuthorizedProjectUpdate class ProjectRecalculateWorker include ApplicationWorker + + data_consistency :always include Gitlab::ExclusiveLeaseHelpers feature_category :authentication_and_authorization @@ -24,7 +26,7 @@ module AuthorizedProjectUpdate private def lock_key(project) - "#{self.class.name.underscore}/#{project.root_namespace.id}" + "#{self.class.name.underscore}/projects/#{project.id}" end end end diff --git a/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb b/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb index 10f7cb20df0..ef5dcc1cb99 100644 --- a/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb +++ b/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb @@ -7,6 +7,7 @@ module AuthorizedProjectUpdate sidekiq_options retry: 3 feature_category :authentication_and_authorization urgency :low + data_consistency :always queue_namespace :authorized_project_update idempotent! diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index a1068117e59..46fe6c7f7ce 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -3,6 +3,8 @@ class AuthorizedProjectsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 prepend WaitableWorker diff --git a/app/workers/auto_devops/disable_worker.rb b/app/workers/auto_devops/disable_worker.rb index 43377382e82..9ec3e5490c2 100644 --- a/app/workers/auto_devops/disable_worker.rb +++ b/app/workers/auto_devops/disable_worker.rb @@ -4,6 +4,8 @@ module AutoDevops class DisableWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include AutoDevopsQueue diff --git a/app/workers/auto_merge_process_worker.rb b/app/workers/auto_merge_process_worker.rb index dda0e970834..976a12f5347 100644 --- a/app/workers/auto_merge_process_worker.rb +++ b/app/workers/auto_merge_process_worker.rb @@ -3,6 +3,8 @@ class AutoMergeProcessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :auto_merge diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index 6b1f10f75b8..ef58258d998 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -3,6 +3,8 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :database diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb index 531e7e5a5fe..3f3d61a8df1 100644 --- a/app/workers/build_success_worker.rb +++ b/app/workers/build_success_worker.rb @@ -3,6 +3,8 @@ class BuildSuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/bulk_import_worker.rb b/app/workers/bulk_import_worker.rb index 25a86ead76e..d7f0b752a34 100644 --- a/app/workers/bulk_import_worker.rb +++ b/app/workers/bulk_import_worker.rb @@ -3,6 +3,8 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + feature_category :importers tags :exclude_from_kubernetes diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb index e7fce112ee1..cc52e349130 100644 --- a/app/workers/bulk_imports/entity_worker.rb +++ b/app/workers/bulk_imports/entity_worker.rb @@ -4,6 +4,8 @@ module BulkImports class EntityWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + feature_category :importers tags :exclude_from_kubernetes diff --git a/app/workers/bulk_imports/export_request_worker.rb b/app/workers/bulk_imports/export_request_worker.rb index d3bb36d830f..d5f7215b08a 100644 --- a/app/workers/bulk_imports/export_request_worker.rb +++ b/app/workers/bulk_imports/export_request_worker.rb @@ -4,6 +4,8 @@ module BulkImports class ExportRequestWorker include ApplicationWorker + data_consistency :always + idempotent! worker_has_external_dependencies! feature_category :importers diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb index d3297017714..713c6c69213 100644 --- a/app/workers/bulk_imports/pipeline_worker.rb +++ b/app/workers/bulk_imports/pipeline_worker.rb @@ -4,6 +4,8 @@ module BulkImports class PipelineWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + NDJSON_PIPELINE_PERFORM_DELAY = 1.minute feature_category :importers diff --git a/app/workers/bulk_imports/relation_export_worker.rb b/app/workers/bulk_imports/relation_export_worker.rb index 9d9449e3a1b..416dad5b3ae 100644 --- a/app/workers/bulk_imports/relation_export_worker.rb +++ b/app/workers/bulk_imports/relation_export_worker.rb @@ -3,6 +3,8 @@ module BulkImports class RelationExportWorker include ApplicationWorker + + data_consistency :always include ExceptionBacktrace idempotent! diff --git a/app/workers/chaos/cpu_spin_worker.rb b/app/workers/chaos/cpu_spin_worker.rb index f8900abc764..ff468d59806 100644 --- a/app/workers/chaos/cpu_spin_worker.rb +++ b/app/workers/chaos/cpu_spin_worker.rb @@ -4,6 +4,8 @@ module Chaos class CpuSpinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ChaosQueue diff --git a/app/workers/chaos/db_spin_worker.rb b/app/workers/chaos/db_spin_worker.rb index 9b5d06414a9..4afe11a137f 100644 --- a/app/workers/chaos/db_spin_worker.rb +++ b/app/workers/chaos/db_spin_worker.rb @@ -4,6 +4,8 @@ module Chaos class DbSpinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ChaosQueue diff --git a/app/workers/chaos/kill_worker.rb b/app/workers/chaos/kill_worker.rb index 4148c139d42..b2f41c513a2 100644 --- a/app/workers/chaos/kill_worker.rb +++ b/app/workers/chaos/kill_worker.rb @@ -3,6 +3,8 @@ module Chaos class KillWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + data_consistency :always include ChaosQueue sidekiq_options retry: false diff --git a/app/workers/chaos/leak_mem_worker.rb b/app/workers/chaos/leak_mem_worker.rb index 788009962db..e040a5ef807 100644 --- a/app/workers/chaos/leak_mem_worker.rb +++ b/app/workers/chaos/leak_mem_worker.rb @@ -4,6 +4,8 @@ module Chaos class LeakMemWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ChaosQueue diff --git a/app/workers/chaos/sleep_worker.rb b/app/workers/chaos/sleep_worker.rb index b9ff5546384..149bab5d9d3 100644 --- a/app/workers/chaos/sleep_worker.rb +++ b/app/workers/chaos/sleep_worker.rb @@ -4,6 +4,8 @@ module Chaos class SleepWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ChaosQueue diff --git a/app/workers/chat_notification_worker.rb b/app/workers/chat_notification_worker.rb index 5fab437f49f..2a2e94cc6f1 100644 --- a/app/workers/chat_notification_worker.rb +++ b/app/workers/chat_notification_worker.rb @@ -3,6 +3,8 @@ class ChatNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + TimeoutExceeded = Class.new(StandardError) sidekiq_options retry: false diff --git a/app/workers/ci/archive_trace_worker.rb b/app/workers/ci/archive_trace_worker.rb index 16288faf370..503cfc07c25 100644 --- a/app/workers/ci/archive_trace_worker.rb +++ b/app/workers/ci/archive_trace_worker.rb @@ -4,6 +4,8 @@ module Ci class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineBackgroundQueue diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb index 5fe3adf870f..12856805243 100644 --- a/app/workers/ci/archive_traces_cron_worker.rb +++ b/app/workers/ci/archive_traces_cron_worker.rb @@ -4,7 +4,8 @@ module Ci class ArchiveTracesCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :continuous_integration diff --git a/app/workers/ci/build_finished_worker.rb b/app/workers/ci/build_finished_worker.rb index 1d6e3b1fa3c..3bca3015988 100644 --- a/app/workers/ci/build_finished_worker.rb +++ b/app/workers/ci/build_finished_worker.rb @@ -4,6 +4,8 @@ module Ci class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue @@ -31,7 +33,6 @@ module Ci # @param [Ci::Build] build The build to process. def process_build(build) # We execute these in sync to reduce IO. - build.parse_trace_sections! build.update_coverage Ci::BuildReportResultService.new.execute(build) diff --git a/app/workers/ci/build_prepare_worker.rb b/app/workers/ci/build_prepare_worker.rb index f30e9d3b885..b6afad55f41 100644 --- a/app/workers/ci/build_prepare_worker.rb +++ b/app/workers/ci/build_prepare_worker.rb @@ -4,6 +4,8 @@ module Ci class BuildPrepareWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index 570f5f28c3d..56a938c32cb 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -4,6 +4,8 @@ module Ci class BuildScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/ci/build_trace_chunk_flush_worker.rb b/app/workers/ci/build_trace_chunk_flush_worker.rb index 1e0da73e08d..79881ec84fd 100644 --- a/app/workers/ci/build_trace_chunk_flush_worker.rb +++ b/app/workers/ci/build_trace_chunk_flush_worker.rb @@ -4,6 +4,8 @@ module Ci class BuildTraceChunkFlushWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineBackgroundQueue diff --git a/app/workers/ci/daily_build_group_report_results_worker.rb b/app/workers/ci/daily_build_group_report_results_worker.rb index b38bef3bcf8..c4d76e4a4d1 100644 --- a/app/workers/ci/daily_build_group_report_results_worker.rb +++ b/app/workers/ci/daily_build_group_report_results_worker.rb @@ -4,6 +4,8 @@ module Ci class DailyBuildGroupReportResultsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineBackgroundQueue diff --git a/app/workers/ci/delete_objects_worker.rb b/app/workers/ci/delete_objects_worker.rb index ff020a3b048..d31d248597b 100644 --- a/app/workers/ci/delete_objects_worker.rb +++ b/app/workers/ci/delete_objects_worker.rb @@ -4,6 +4,8 @@ module Ci class DeleteObjectsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include LimitedCapacity::Worker diff --git a/app/workers/ci/delete_unit_tests_worker.rb b/app/workers/ci/delete_unit_tests_worker.rb index ddfc70c43d4..d5bb72ce80c 100644 --- a/app/workers/ci/delete_unit_tests_worker.rb +++ b/app/workers/ci/delete_unit_tests_worker.rb @@ -3,6 +3,8 @@ module Ci class DeleteUnitTestsWorker include ApplicationWorker + + data_consistency :always # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/ci/drop_pipeline_worker.rb b/app/workers/ci/drop_pipeline_worker.rb index bc158433228..f3672dba3fe 100644 --- a/app/workers/ci/drop_pipeline_worker.rb +++ b/app/workers/ci/drop_pipeline_worker.rb @@ -4,6 +4,8 @@ module Ci class DropPipelineWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/ci/initial_pipeline_process_worker.rb b/app/workers/ci/initial_pipeline_process_worker.rb index ca41a7fb577..8d7a62e5b09 100644 --- a/app/workers/ci/initial_pipeline_process_worker.rb +++ b/app/workers/ci/initial_pipeline_process_worker.rb @@ -4,6 +4,8 @@ module Ci class InitialPipelineProcessWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb b/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb index bd061b5f988..af042dc1e64 100644 --- a/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb +++ b/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb @@ -4,6 +4,8 @@ module Ci class AddTodoWhenBuildFailsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb b/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb index ec0cb69d0c7..16c4744eae1 100644 --- a/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb +++ b/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb @@ -5,6 +5,8 @@ module Ci class CoverageReportWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineBackgroundQueue diff --git a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb index 558153c69b2..06bc100c66a 100644 --- a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb +++ b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb @@ -5,6 +5,8 @@ module Ci class CreateQualityReportWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :pipeline_background diff --git a/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb b/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb index 004c1d444a2..e4dc293353b 100644 --- a/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb +++ b/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb @@ -5,7 +5,8 @@ module Ci class ExpireArtifactsWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb b/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb index b0921f6e10b..160947fb38e 100644 --- a/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb +++ b/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb @@ -4,6 +4,8 @@ module Ci class PipelineSuccessUnlockArtifactsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineBackgroundQueue diff --git a/app/workers/ci/ref_delete_unlock_artifacts_worker.rb b/app/workers/ci/ref_delete_unlock_artifacts_worker.rb index d20c501100e..0c217644cc4 100644 --- a/app/workers/ci/ref_delete_unlock_artifacts_worker.rb +++ b/app/workers/ci/ref_delete_unlock_artifacts_worker.rb @@ -4,6 +4,8 @@ module Ci class RefDeleteUnlockArtifactsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineBackgroundQueue diff --git a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb index ad0ed3d16f1..98b4f4ad73a 100644 --- a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb +++ b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb @@ -8,6 +8,8 @@ module Ci class AssignResourceFromResourceGroupWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/ci/schedule_delete_objects_cron_worker.rb b/app/workers/ci/schedule_delete_objects_cron_worker.rb index 6489665fafd..06bf83ae0a7 100644 --- a/app/workers/ci/schedule_delete_objects_cron_worker.rb +++ b/app/workers/ci/schedule_delete_objects_cron_worker.rb @@ -4,7 +4,8 @@ module Ci class ScheduleDeleteObjectsCronWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/ci/test_failure_history_worker.rb b/app/workers/ci/test_failure_history_worker.rb index 3937f720788..b67797edf0b 100644 --- a/app/workers/ci/test_failure_history_worker.rb +++ b/app/workers/ci/test_failure_history_worker.rb @@ -4,6 +4,8 @@ module Ci class TestFailureHistoryWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineBackgroundQueue diff --git a/app/workers/ci_platform_metrics_update_cron_worker.rb b/app/workers/ci_platform_metrics_update_cron_worker.rb index 05af0a0a73b..8de6e8a1253 100644 --- a/app/workers/ci_platform_metrics_update_cron_worker.rb +++ b/app/workers/ci_platform_metrics_update_cron_worker.rb @@ -3,7 +3,7 @@ class CiPlatformMetricsUpdateCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always # This worker does not perform work scoped to a context include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb index a8de8efbce6..9adc026ced2 100644 --- a/app/workers/cleanup_container_repository_worker.rb +++ b/app/workers/cleanup_container_repository_worker.rb @@ -3,6 +3,8 @@ class CleanupContainerRepositoryWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :container_repository diff --git a/app/workers/cluster_configure_istio_worker.rb b/app/workers/cluster_configure_istio_worker.rb index 07c032da838..0def66b7381 100644 --- a/app/workers/cluster_configure_istio_worker.rb +++ b/app/workers/cluster_configure_istio_worker.rb @@ -3,6 +3,8 @@ class ClusterConfigureIstioWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb index 71374de19f5..e16e6e9ca71 100644 --- a/app/workers/cluster_install_app_worker.rb +++ b/app/workers/cluster_install_app_worker.rb @@ -3,6 +3,8 @@ class ClusterInstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_patch_app_worker.rb b/app/workers/cluster_patch_app_worker.rb index 674a565f7f7..bb16cf7a5e6 100644 --- a/app/workers/cluster_patch_app_worker.rb +++ b/app/workers/cluster_patch_app_worker.rb @@ -3,6 +3,8 @@ class ClusterPatchAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb index 142ad84f746..04c9174347f 100644 --- a/app/workers/cluster_provision_worker.rb +++ b/app/workers/cluster_provision_worker.rb @@ -3,6 +3,8 @@ class ClusterProvisionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue diff --git a/app/workers/cluster_update_app_worker.rb b/app/workers/cluster_update_app_worker.rb index dc57a1a90d9..cd2b2e38ea4 100644 --- a/app/workers/cluster_update_app_worker.rb +++ b/app/workers/cluster_update_app_worker.rb @@ -5,6 +5,8 @@ class ClusterUpdateAppWorker # rubocop:disable Scalability/IdempotentWorker UpdateAlreadyInProgressError = Class.new(StandardError) include ApplicationWorker + + data_consistency :always include ClusterQueue include ClusterApplications include ExclusiveLeaseGuard diff --git a/app/workers/cluster_upgrade_app_worker.rb b/app/workers/cluster_upgrade_app_worker.rb index 909ada2044f..bbe0cb7f0c2 100644 --- a/app/workers/cluster_upgrade_app_worker.rb +++ b/app/workers/cluster_upgrade_app_worker.rb @@ -3,6 +3,8 @@ class ClusterUpgradeAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb index 19e33cd17b0..846e4442233 100644 --- a/app/workers/cluster_wait_for_app_installation_worker.rb +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -3,6 +3,8 @@ class ClusterWaitForAppInstallationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_app_update_worker.rb b/app/workers/cluster_wait_for_app_update_worker.rb index 185959884a1..e96e03ae249 100644 --- a/app/workers/cluster_wait_for_app_update_worker.rb +++ b/app/workers/cluster_wait_for_app_update_worker.rb @@ -3,6 +3,8 @@ class ClusterWaitForAppUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb index 4a010c749a2..561e72562e9 100644 --- a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb +++ b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb @@ -3,6 +3,8 @@ class ClusterWaitForIngressIpAddressWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/clusters/applications/activate_service_worker.rb b/app/workers/clusters/applications/activate_service_worker.rb index a7073b78a81..55e224887f4 100644 --- a/app/workers/clusters/applications/activate_service_worker.rb +++ b/app/workers/clusters/applications/activate_service_worker.rb @@ -5,6 +5,8 @@ module Clusters class ActivateServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue diff --git a/app/workers/clusters/applications/check_prometheus_health_worker.rb b/app/workers/clusters/applications/check_prometheus_health_worker.rb index 4db7314cbc0..4f85c5c5b7a 100644 --- a/app/workers/clusters/applications/check_prometheus_health_worker.rb +++ b/app/workers/clusters/applications/check_prometheus_health_worker.rb @@ -5,7 +5,8 @@ module Clusters class CheckPrometheusHealthWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/clusters/applications/deactivate_service_worker.rb b/app/workers/clusters/applications/deactivate_service_worker.rb index 9337af56623..4c8d21a7c4d 100644 --- a/app/workers/clusters/applications/deactivate_service_worker.rb +++ b/app/workers/clusters/applications/deactivate_service_worker.rb @@ -5,6 +5,8 @@ module Clusters class DeactivateServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue @@ -14,9 +16,11 @@ module Clusters cluster = Clusters::Cluster.find_by_id(cluster_id) raise cluster_missing_error(integration_name) unless cluster - integration = ::Project.integration_association_name(integration_name).to_sym - cluster.all_projects.with_integration(integration).find_each do |project| - project.public_send(integration).update!(active: false) # rubocop:disable GitlabSecurity/PublicSend + integration_class = Integration.integration_name_to_model(integration_name) + integration_association_name = ::Project.integration_association_name(integration_name).to_sym + + cluster.all_projects.with_integration(integration_class).include_integration(integration_association_name).find_each do |project| + project.public_send(integration_association_name).update!(active: false) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/workers/clusters/applications/uninstall_worker.rb b/app/workers/clusters/applications/uninstall_worker.rb index 3a4564ca7ab..da290eaf1f6 100644 --- a/app/workers/clusters/applications/uninstall_worker.rb +++ b/app/workers/clusters/applications/uninstall_worker.rb @@ -5,6 +5,8 @@ module Clusters class UninstallWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb index 18801ad7e64..510ea8e7b17 100644 --- a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb +++ b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb @@ -5,6 +5,8 @@ module Clusters class WaitForUninstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index e158ae0c298..6cc6c30c5e9 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -54,6 +54,10 @@ module ApplicationWorker subclass.after_set_class_attribute { subclass.set_queue } end + def generated_queue_name + Gitlab::SidekiqConfig::WorkerRouter.queue_name_from_worker_name(self) + end + override :validate_worker_attributes! def validate_worker_attributes! super diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index 1eff53cea01..a377b7a2000 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -17,10 +17,6 @@ module Gitlab feature_category :importers worker_has_external_dependencies! - - def logger - @logger ||= Gitlab::Import::Logger.build - end end # project - An instance of `Project` to import the data into. @@ -39,8 +35,24 @@ module Gitlab Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :imported) info(project.id, message: 'importer finished') + rescue KeyError => e + # This exception will be more useful in development when a new + # Representation is created but the developer forgot to add a + # `:github_id` field. + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: importer_class.name, + exception: e, + fail_import: true + ) + + raise(e) rescue StandardError => e - error(project.id, e, hash) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: importer_class.name, + exception: e + ) end def object_type @@ -63,28 +75,11 @@ module Gitlab attr_accessor :github_id def info(project_id, extra = {}) - logger.info(log_attributes(project_id, extra)) - end - - def error(project_id, exception, data = {}) - logger.error( - log_attributes( - project_id, - message: 'importer failed', - 'error.message': exception.message, - 'github.data': data - ) - ) - - Gitlab::ErrorTracking.track_and_raise_exception( - exception, - log_attributes(project_id) - ) + Logger.info(log_attributes(project_id, extra)) end def log_attributes(project_id, extra = {}) extra.merge( - import_source: :github, project_id: project_id, importer: importer_class.name, github_id: github_id diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb index 05eb7fbc2cb..e7156ac12f8 100644 --- a/app/workers/concerns/gitlab/github_import/queue.rb +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -17,13 +17,10 @@ module Gitlab sidekiq_options dead: false, retry: 5 sidekiq_retries_exhausted do |msg, e| - Gitlab::Import::Logger.error( - event: :github_importer_exhausted, - message: msg['error_message'], - class: msg['class'], - args: msg['args'], - exception_message: e.message, - exception_backtrace: e.backtrace + Gitlab::Import::ImportFailureService.track( + project_id: msg['args'][0], + exception: e, + fail_import: true ) end end diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index 916b273a28f..d7b4578af63 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -15,7 +15,14 @@ module Gitlab info(project_id, message: 'stage finished') rescue StandardError => e - error(project_id, e) + Gitlab::Import::ImportFailureService.track( + project_id: project_id, + exception: e, + error_source: self.class.name, + fail_import: abort_on_failure + ) + + raise(e) end # client - An instance of Gitlab::GithubImport::Client. @@ -34,38 +41,22 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - private - - def info(project_id, extra = {}) - logger.info(log_attributes(project_id, extra)) + def abort_on_failure + false end - def error(project_id, exception) - logger.error( - log_attributes( - project_id, - message: 'stage failed', - 'error.message': exception.message - ) - ) + private - Gitlab::ErrorTracking.track_and_raise_exception( - exception, - log_attributes(project_id) - ) + def info(project_id, extra = {}) + Gitlab::GithubImport::Logger.info(log_attributes(project_id, extra)) end def log_attributes(project_id, extra = {}) extra.merge( - import_source: :github, project_id: project_id, import_stage: self.class.name ) end - - def logger - @logger ||= Gitlab::Import::Logger.build - end end end end diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb index 33dda6a8f0c..de9bb4d5a93 100644 --- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb +++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb @@ -4,6 +4,8 @@ module ContainerExpirationPolicies class CleanupContainerRepositoryWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include LimitedCapacity::Worker include Gitlab::Utils::StrongMemoize @@ -61,16 +63,23 @@ module ContainerExpirationPolicies def container_repository strong_memoize(:container_repository) do ContainerRepository.transaction do - # We need a lock to prevent two workers from picking up the same row - container_repository = next_container_repository + repository = next_container_repository + + repository&.tap do |repo| + log_info( + project_id: repo.project_id, + container_repository_id: repo.id + ) - container_repository&.tap(&:cleanup_ongoing!) + repo.cleanup_ongoing! + end end end end def next_container_repository # rubocop: disable CodeReuse/ActiveRecord + # We need a lock to prevent two workers from picking up the same row next_one_requiring = ContainerRepository.requiring_cleanup .order(:expiration_policy_cleanup_status, :expiration_policy_started_at) .limit(1) diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb index a35ca5d184e..a791fe5d350 100644 --- a/app/workers/container_expiration_policy_worker.rb +++ b/app/workers/container_expiration_policy_worker.rb @@ -3,7 +3,8 @@ class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue include ExclusiveLeaseGuard diff --git a/app/workers/create_commit_signature_worker.rb b/app/workers/create_commit_signature_worker.rb index 0ba2cc41e99..01a2e109967 100644 --- a/app/workers/create_commit_signature_worker.rb +++ b/app/workers/create_commit_signature_worker.rb @@ -3,6 +3,8 @@ class CreateCommitSignatureWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/create_note_diff_file_worker.rb b/app/workers/create_note_diff_file_worker.rb index 0af203fc3bd..4bea4fc872e 100644 --- a/app/workers/create_note_diff_file_worker.rb +++ b/app/workers/create_note_diff_file_worker.rb @@ -3,6 +3,8 @@ class CreateNoteDiffFileWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review diff --git a/app/workers/create_pipeline_worker.rb b/app/workers/create_pipeline_worker.rb index a9072e1661f..eb02fe72294 100644 --- a/app/workers/create_pipeline_worker.rb +++ b/app/workers/create_pipeline_worker.rb @@ -3,6 +3,8 @@ class CreatePipelineWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/database/batched_background_migration_worker.rb b/app/workers/database/batched_background_migration_worker.rb index 5a326a351e8..0750ff1acaf 100644 --- a/app/workers/database/batched_background_migration_worker.rb +++ b/app/workers/database/batched_background_migration_worker.rb @@ -4,7 +4,8 @@ module Database class BatchedBackgroundMigrationWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :database diff --git a/app/workers/database/drop_detached_partitions_worker.rb b/app/workers/database/drop_detached_partitions_worker.rb new file mode 100644 index 00000000000..f9c8ce57a36 --- /dev/null +++ b/app/workers/database/drop_detached_partitions_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Database + class DropDetachedPartitionsWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :database + data_consistency :always + idempotent! + + def perform + Gitlab::Database::Partitioning::DetachedPartitionDropper.new.perform + ensure + Gitlab::Database::Partitioning::PartitionMonitoring.new.report_metrics + end + end +end diff --git a/app/workers/database/partition_management_worker.rb b/app/workers/database/partition_management_worker.rb index c9b1cd6d261..a203c76558a 100644 --- a/app/workers/database/partition_management_worker.rb +++ b/app/workers/database/partition_management_worker.rb @@ -4,9 +4,10 @@ module Database class PartitionManagementWorker include ApplicationWorker - sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + data_consistency :always + feature_category :database idempotent! diff --git a/app/workers/delete_container_repository_worker.rb b/app/workers/delete_container_repository_worker.rb index f39f8bf44a4..beeca559060 100644 --- a/app/workers/delete_container_repository_worker.rb +++ b/app/workers/delete_container_repository_worker.rb @@ -3,6 +3,8 @@ class DeleteContainerRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExclusiveLeaseGuard diff --git a/app/workers/delete_diff_files_worker.rb b/app/workers/delete_diff_files_worker.rb index 46dac5d8d39..54d8fcb6dfd 100644 --- a/app/workers/delete_diff_files_worker.rb +++ b/app/workers/delete_diff_files_worker.rb @@ -3,6 +3,8 @@ class DeleteDiffFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review diff --git a/app/workers/delete_merged_branches_worker.rb b/app/workers/delete_merged_branches_worker.rb index c7e1a4da965..5e1103dbc5d 100644 --- a/app/workers/delete_merged_branches_worker.rb +++ b/app/workers/delete_merged_branches_worker.rb @@ -3,6 +3,8 @@ class DeleteMergedBranchesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/delete_stored_files_worker.rb b/app/workers/delete_stored_files_worker.rb index 75113b4787c..d1080c8df64 100644 --- a/app/workers/delete_stored_files_worker.rb +++ b/app/workers/delete_stored_files_worker.rb @@ -3,6 +3,8 @@ class DeleteStoredFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category_not_owned! diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb index f1b9f859ce6..0af084caf86 100644 --- a/app/workers/delete_user_worker.rb +++ b/app/workers/delete_user_worker.rb @@ -3,6 +3,8 @@ class DeleteUserWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :authentication_and_authorization diff --git a/app/workers/deployments/drop_older_deployments_worker.rb b/app/workers/deployments/drop_older_deployments_worker.rb index 6ca819e7942..979f683cfb3 100644 --- a/app/workers/deployments/drop_older_deployments_worker.rb +++ b/app/workers/deployments/drop_older_deployments_worker.rb @@ -4,6 +4,8 @@ module Deployments class DropOlderDeploymentsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :deployment diff --git a/app/workers/deployments/finished_worker.rb b/app/workers/deployments/finished_worker.rb index 3de06c381cd..25121656408 100644 --- a/app/workers/deployments/finished_worker.rb +++ b/app/workers/deployments/finished_worker.rb @@ -6,6 +6,8 @@ module Deployments class FinishedWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :deployment diff --git a/app/workers/deployments/forward_deployment_worker.rb b/app/workers/deployments/forward_deployment_worker.rb index 946945051ba..7f5eb13b88d 100644 --- a/app/workers/deployments/forward_deployment_worker.rb +++ b/app/workers/deployments/forward_deployment_worker.rb @@ -6,6 +6,8 @@ module Deployments class ForwardDeploymentWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :deployment diff --git a/app/workers/deployments/hooks_worker.rb b/app/workers/deployments/hooks_worker.rb index beac44881fb..d23a440ed36 100644 --- a/app/workers/deployments/hooks_worker.rb +++ b/app/workers/deployments/hooks_worker.rb @@ -4,6 +4,8 @@ module Deployments class HooksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :delayed, feature_flag: :load_balancing_for_deployments_hooks_worker + queue_namespace :deployment feature_category :continuous_delivery diff --git a/app/workers/deployments/link_merge_request_worker.rb b/app/workers/deployments/link_merge_request_worker.rb index 70947b3f731..81eeefcb248 100644 --- a/app/workers/deployments/link_merge_request_worker.rb +++ b/app/workers/deployments/link_merge_request_worker.rb @@ -4,6 +4,8 @@ module Deployments class LinkMergeRequestWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :deployment diff --git a/app/workers/deployments/success_worker.rb b/app/workers/deployments/success_worker.rb index eab331433e8..401c2d7600c 100644 --- a/app/workers/deployments/success_worker.rb +++ b/app/workers/deployments/success_worker.rb @@ -6,6 +6,8 @@ module Deployments class SuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :deployment diff --git a/app/workers/deployments/update_environment_worker.rb b/app/workers/deployments/update_environment_worker.rb index 5c71a13064e..2cd9eec754c 100644 --- a/app/workers/deployments/update_environment_worker.rb +++ b/app/workers/deployments/update_environment_worker.rb @@ -4,6 +4,8 @@ module Deployments class UpdateEnvironmentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :deployment diff --git a/app/workers/design_management/copy_design_collection_worker.rb b/app/workers/design_management/copy_design_collection_worker.rb index 28b511c7c27..a498eed173c 100644 --- a/app/workers/design_management/copy_design_collection_worker.rb +++ b/app/workers/design_management/copy_design_collection_worker.rb @@ -4,6 +4,8 @@ module DesignManagement class CopyDesignCollectionWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :design_management diff --git a/app/workers/design_management/new_version_worker.rb b/app/workers/design_management/new_version_worker.rb index eee96858c34..c3de64b841b 100644 --- a/app/workers/design_management/new_version_worker.rb +++ b/app/workers/design_management/new_version_worker.rb @@ -4,6 +4,8 @@ module DesignManagement class NewVersionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :design_management diff --git a/app/workers/destroy_pages_deployments_worker.rb b/app/workers/destroy_pages_deployments_worker.rb index edd446628aa..36424f7473e 100644 --- a/app/workers/destroy_pages_deployments_worker.rb +++ b/app/workers/destroy_pages_deployments_worker.rb @@ -3,6 +3,8 @@ class DestroyPagesDeploymentsWorker include ApplicationWorker + data_consistency :always + idempotent! loggable_arguments 0, 1 diff --git a/app/workers/detect_repository_languages_worker.rb b/app/workers/detect_repository_languages_worker.rb index ef66287a692..8c7ef6c9e32 100644 --- a/app/workers/detect_repository_languages_worker.rb +++ b/app/workers/detect_repository_languages_worker.rb @@ -2,6 +2,8 @@ class DetectRepositoryLanguagesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + data_consistency :always include ExceptionBacktrace include ExclusiveLeaseGuard diff --git a/app/workers/disallow_two_factor_for_group_worker.rb b/app/workers/disallow_two_factor_for_group_worker.rb index 3a48e3ab5da..4f5ef69a730 100644 --- a/app/workers/disallow_two_factor_for_group_worker.rb +++ b/app/workers/disallow_two_factor_for_group_worker.rb @@ -3,6 +3,8 @@ class DisallowTwoFactorForGroupWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExceptionBacktrace diff --git a/app/workers/disallow_two_factor_for_subgroups_worker.rb b/app/workers/disallow_two_factor_for_subgroups_worker.rb index f5b31e0bcf0..d3528b0674b 100644 --- a/app/workers/disallow_two_factor_for_subgroups_worker.rb +++ b/app/workers/disallow_two_factor_for_subgroups_worker.rb @@ -3,6 +3,8 @@ class DisallowTwoFactorForSubgroupsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExceptionBacktrace diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 37ed1001c9d..1514897b2e4 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -3,12 +3,17 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :issue_tracking urgency :high weight 2 + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + tags :needs_own_queue + attr_accessor :raw def perform(raw) diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index 9c4418c5f31..0230a226567 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -3,6 +3,8 @@ class EmailsOnPushWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 attr_reader :email, :skip_premailer diff --git a/app/workers/environments/auto_delete_cron_worker.rb b/app/workers/environments/auto_delete_cron_worker.rb new file mode 100644 index 00000000000..12c08f07d95 --- /dev/null +++ b/app/workers/environments/auto_delete_cron_worker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Environments + class AutoDeleteCronWorker + include ApplicationWorker + include ::Gitlab::LoopHelpers + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + data_consistency :always + feature_category :continuous_delivery + deduplicate :until_executed, including_scheduled: true + idempotent! + + LOOP_TIMEOUT = 45.minutes + LOOP_LIMIT = 1000 + BATCH_SIZE = 100 + + def perform + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + destroy_in_batch + end + end + + private + + def destroy_in_batch + environments = Environment.auto_deletable(BATCH_SIZE) + + return false if environments.empty? + + environments.each(&:destroy) + end + end +end diff --git a/app/workers/environments/auto_stop_cron_worker.rb b/app/workers/environments/auto_stop_cron_worker.rb index 64028839df1..4d6453a85e7 100644 --- a/app/workers/environments/auto_stop_cron_worker.rb +++ b/app/workers/environments/auto_stop_cron_worker.rb @@ -4,10 +4,12 @@ module Environments class AutoStopCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :continuous_delivery + worker_resource_boundary :cpu def perform AutoStopService.new.execute diff --git a/app/workers/environments/canary_ingress/update_worker.rb b/app/workers/environments/canary_ingress/update_worker.rb index ecdfc6f0581..591c88cac96 100644 --- a/app/workers/environments/canary_ingress/update_worker.rb +++ b/app/workers/environments/canary_ingress/update_worker.rb @@ -5,6 +5,8 @@ module Environments class UpdateWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: false idempotent! worker_has_external_dependencies! diff --git a/app/workers/error_tracking_issue_link_worker.rb b/app/workers/error_tracking_issue_link_worker.rb index 6c5a96822a6..37a3a1e7bcb 100644 --- a/app/workers/error_tracking_issue_link_worker.rb +++ b/app/workers/error_tracking_issue_link_worker.rb @@ -8,6 +8,8 @@ class ErrorTrackingIssueLinkWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExclusiveLeaseGuard include Gitlab::Utils::StrongMemoize diff --git a/app/workers/experiments/record_conversion_event_worker.rb b/app/workers/experiments/record_conversion_event_worker.rb index 9fc76a2173b..4c82c114d15 100644 --- a/app/workers/experiments/record_conversion_event_worker.rb +++ b/app/workers/experiments/record_conversion_event_worker.rb @@ -4,6 +4,8 @@ module Experiments class RecordConversionEventWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :users diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb index a9fa94ef301..65d387f73ed 100644 --- a/app/workers/expire_build_artifacts_worker.rb +++ b/app/workers/expire_build_artifacts_worker.rb @@ -3,7 +3,8 @@ class ExpireBuildArtifactsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index 3e6e81867bd..96378acca08 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -3,6 +3,8 @@ class ExpireBuildInstanceArtifactsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :continuous_integration diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index 074c35997f6..cd5ca25f031 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -3,6 +3,8 @@ class ExpireJobCacheWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/export_csv_worker.rb b/app/workers/export_csv_worker.rb index a2ad0cb92fd..68feaa61cdd 100644 --- a/app/workers/export_csv_worker.rb +++ b/app/workers/export_csv_worker.rb @@ -3,6 +3,8 @@ class ExportCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :issue_tracking diff --git a/app/workers/file_hook_worker.rb b/app/workers/file_hook_worker.rb index b1422cd8795..77aaf957254 100644 --- a/app/workers/file_hook_worker.rb +++ b/app/workers/file_hook_worker.rb @@ -3,9 +3,11 @@ class FileHookWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: false feature_category :integrations loggable_arguments 0 + urgency :low def perform(file_name, data) success, message = Gitlab::FileHook.execute(file_name, data) diff --git a/app/workers/flush_counter_increments_worker.rb b/app/workers/flush_counter_increments_worker.rb index 44d30b4ba3d..bcb6a4c2bca 100644 --- a/app/workers/flush_counter_increments_worker.rb +++ b/app/workers/flush_counter_increments_worker.rb @@ -8,6 +8,8 @@ class FlushCounterIncrementsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category_not_owned! diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index f25296f0461..06f0ef623c2 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -9,6 +9,8 @@ module Gitlab class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ::Gitlab::Import::AdvanceStage diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb index 91dab3470d9..cce179542c7 100644 --- a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -6,6 +6,7 @@ module Gitlab include ObjectImporter tags :exclude_from_kubernetes + worker_resource_boundary :cpu def representation_class Gitlab::GithubImport::Representation::PullRequest diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb index de10fe40589..8796d6392df 100644 --- a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -6,6 +6,7 @@ module Gitlab include ObjectImporter tags :exclude_from_kubernetes + worker_resource_boundary :cpu def representation_class Gitlab::GithubImport::Representation::PullRequestReview diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb index 1c769921ab3..2b9fb26d53a 100644 --- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb +++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb @@ -5,6 +5,8 @@ module Gitlab class RefreshImportJidWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index f909d7e2f34..006b79dbff4 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -6,6 +6,8 @@ module Gitlab class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index 7ca23ecad20..715c39caf42 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportBaseDataWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index d66698277b0..c33836e20d1 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportIssuesAndDiffNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb index 2a66a08d534..f6f5687130f 100644 --- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 873e389fca6..0160145ffe2 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb index 5743648680d..7d83fe288da 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportPullRequestsMergedByWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb index 532d550f190..ea3b89efd22 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportPullRequestsReviewsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index 5755aea21ce..d76d36531d1 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index e113563ce8b..227b7c304b0 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods @@ -26,7 +28,7 @@ module Gitlab info(project.id, message: "starting importer", importer: 'Importer::RepositoryImporter') importer = Importer::RepositoryImporter.new(project, client) - return unless importer.execute + importer.execute counter.increment @@ -39,6 +41,10 @@ module Gitlab 'The number of imported GitHub repositories' ) end + + def abort_on_failure + true + end end end end diff --git a/app/workers/gitlab/import/stuck_import_job.rb b/app/workers/gitlab/import/stuck_import_job.rb index 57fb3baf2b5..efbea7d8133 100644 --- a/app/workers/gitlab/import/stuck_import_job.rb +++ b/app/workers/gitlab/import/stuck_import_job.rb @@ -5,12 +5,13 @@ module Gitlab module StuckImportJob extend ActiveSupport::Concern + StuckImportJobError = Class.new(StandardError) + IMPORT_JOBS_EXPIRATION = 24.hours.seconds.to_i included do include ApplicationWorker - sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker updates several import states inline and does not schedule # other jobs. So no context needed @@ -35,9 +36,9 @@ module Gitlab end def mark_imports_without_jid_as_failed! - enqueued_import_states_without_jid.each do |import_state| - import_state.mark_as_failed(error_message) - end.size + enqueued_import_states_without_jid + .each(&method(:mark_as_failed)) + .size end def mark_imports_with_jid_as_failed! @@ -59,9 +60,20 @@ module Gitlab job_ids: completed_import_state_jids ) - completed_import_states.each do |import_state| - import_state.mark_as_failed(error_message) - end.size + completed_import_states + .each(&method(:mark_as_failed)) + .size + end + + def mark_as_failed(import_state) + raise StuckImportJobError, error_message + rescue StuckImportJobError => e + Gitlab::Import::ImportFailureService.track( + import_state: import_state, + exception: e, + error_source: self.class.name, + fail_import: true + ) end def enqueued_import_states diff --git a/app/workers/gitlab/jira_import/advance_stage_worker.rb b/app/workers/gitlab/jira_import/advance_stage_worker.rb index 6387054d448..5fae7caf791 100644 --- a/app/workers/gitlab/jira_import/advance_stage_worker.rb +++ b/app/workers/gitlab/jira_import/advance_stage_worker.rb @@ -5,6 +5,8 @@ module Gitlab class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include QueueOptions include ::Gitlab::Import::AdvanceStage diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index 98bde2218c2..eabe7328b92 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -5,6 +5,8 @@ module Gitlab class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include NotifyUponDeath include Gitlab::JiraImport::QueueOptions @@ -52,7 +54,7 @@ module Gitlab label_link_attrs << build_label_attrs(issue_id, import_label_id.to_i) - Gitlab::Database.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert end def assign_issue(project_id, issue_id, assignee_ids) @@ -60,7 +62,7 @@ module Gitlab assignee_attrs = assignee_ids.map { |user_id| { issue_id: issue_id, user_id: user_id } } - Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert end def build_label_attrs(issue_id, label_id) diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb index e327ced8c65..e0de3ee169e 100644 --- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -6,6 +6,8 @@ module Gitlab class StartImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ProjectStartImport include ProjectImportOptions diff --git a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb index 867a12fbac2..f650681fc2f 100644 --- a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb +++ b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb @@ -4,6 +4,8 @@ module Gitlab class ImportTasksWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ProjectImportOptions # This marks the project as failed after too many tries diff --git a/app/workers/gitlab_performance_bar_stats_worker.rb b/app/workers/gitlab_performance_bar_stats_worker.rb index 4f7fdcf96f0..4e8bcb9af7b 100644 --- a/app/workers/gitlab_performance_bar_stats_worker.rb +++ b/app/workers/gitlab_performance_bar_stats_worker.rb @@ -3,6 +3,9 @@ class GitlabPerformanceBarStatsWorker include ApplicationWorker + data_consistency :always + worker_resource_boundary :cpu + sidekiq_options retry: 3 LEASE_KEY = 'gitlab:performance_bar_stats' diff --git a/app/workers/gitlab_service_ping_worker.rb b/app/workers/gitlab_service_ping_worker.rb index a27629eac0a..6cf46458b1e 100644 --- a/app/workers/gitlab_service_ping_worker.rb +++ b/app/workers/gitlab_service_ping_worker.rb @@ -5,10 +5,13 @@ class GitlabServicePingWorker # rubocop:disable Scalability/IdempotentWorker LEASE_TIMEOUT = 86400 include ApplicationWorker + + data_consistency :always include CronjobQueue # rubocop:disable Scalability/CronWorkerContext include Gitlab::ExclusiveLeaseHelpers feature_category :service_ping + worker_resource_boundary :cpu sidekiq_options retry: 3, dead: false sidekiq_retry_in { |count| (count + 1) * 8.hours.to_i } diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index de1e9af7bae..1bcaf5a42be 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -3,6 +3,8 @@ class GitlabShellWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include Gitlab::ShellAdapter diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb index 2c140c89e26..f44c109f12d 100644 --- a/app/workers/group_destroy_worker.rb +++ b/app/workers/group_destroy_worker.rb @@ -3,6 +3,8 @@ class GroupDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExceptionBacktrace diff --git a/app/workers/group_export_worker.rb b/app/workers/group_export_worker.rb index a212147d8fd..f6f9a69fb17 100644 --- a/app/workers/group_export_worker.rb +++ b/app/workers/group_export_worker.rb @@ -2,6 +2,8 @@ class GroupExportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + data_consistency :always include ExceptionBacktrace feature_category :importers diff --git a/app/workers/group_import_worker.rb b/app/workers/group_import_worker.rb index b8b596f459b..198c6274166 100644 --- a/app/workers/group_import_worker.rb +++ b/app/workers/group_import_worker.rb @@ -3,6 +3,8 @@ class GroupImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: false, dead: false feature_category :importers diff --git a/app/workers/hashed_storage/migrator_worker.rb b/app/workers/hashed_storage/migrator_worker.rb index 735d8a2447a..80e86fd7814 100644 --- a/app/workers/hashed_storage/migrator_worker.rb +++ b/app/workers/hashed_storage/migrator_worker.rb @@ -4,11 +4,16 @@ module HashedStorage class MigratorWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :hashed_storage feature_category :source_code_management - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#migration_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index 0659c8a6a46..edddea55356 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -4,11 +4,16 @@ module HashedStorage class ProjectMigrateWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :hashed_storage loggable_arguments 1 - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#migration_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index a5ee8b35176..c5841dbbb28 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -4,11 +4,16 @@ module HashedStorage class ProjectRollbackWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :hashed_storage loggable_arguments 1 - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/rollbacker_worker.rb b/app/workers/hashed_storage/rollbacker_worker.rb index 447bdfa6220..90e48f0e37a 100644 --- a/app/workers/hashed_storage/rollbacker_worker.rb +++ b/app/workers/hashed_storage/rollbacker_worker.rb @@ -4,11 +4,16 @@ module HashedStorage class RollbackerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :hashed_storage feature_category :source_code_management - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/import_export_project_cleanup_worker.rb b/app/workers/import_export_project_cleanup_worker.rb index 6e112a47932..4cb59c424d1 100644 --- a/app/workers/import_export_project_cleanup_worker.rb +++ b/app/workers/import_export_project_cleanup_worker.rb @@ -3,7 +3,8 @@ class ImportExportProjectCleanupWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb index 46b59dc398f..58e411c7b19 100644 --- a/app/workers/import_issues_csv_worker.rb +++ b/app/workers/import_issues_csv_worker.rb @@ -3,6 +3,8 @@ class ImportIssuesCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 idempotent! diff --git a/app/workers/incident_management/add_severity_system_note_worker.rb b/app/workers/incident_management/add_severity_system_note_worker.rb index 62ed902e488..31da7b0bcfe 100644 --- a/app/workers/incident_management/add_severity_system_note_worker.rb +++ b/app/workers/incident_management/add_severity_system_note_worker.rb @@ -4,6 +4,9 @@ module IncidentManagement class AddSeveritySystemNoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + worker_resource_boundary :cpu + sidekiq_options retry: 3 queue_namespace :incident_management diff --git a/app/workers/incident_management/pager_duty/process_incident_worker.rb b/app/workers/incident_management/pager_duty/process_incident_worker.rb index 413a297a024..933d8e12d25 100644 --- a/app/workers/incident_management/pager_duty/process_incident_worker.rb +++ b/app/workers/incident_management/pager_duty/process_incident_worker.rb @@ -5,6 +5,8 @@ module IncidentManagement class ProcessIncidentWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :incident_management diff --git a/app/workers/incident_management/process_alert_worker_v2.rb b/app/workers/incident_management/process_alert_worker_v2.rb index 04bf6970578..f3049560bcd 100644 --- a/app/workers/incident_management/process_alert_worker_v2.rb +++ b/app/workers/incident_management/process_alert_worker_v2.rb @@ -4,6 +4,9 @@ module IncidentManagement class ProcessAlertWorkerV2 # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + worker_resource_boundary :cpu + queue_namespace :incident_management feature_category :incident_management diff --git a/app/workers/invalid_gpg_signature_update_worker.rb b/app/workers/invalid_gpg_signature_update_worker.rb index 662817b5a92..c9567e102d3 100644 --- a/app/workers/invalid_gpg_signature_update_worker.rb +++ b/app/workers/invalid_gpg_signature_update_worker.rb @@ -3,6 +3,8 @@ class InvalidGpgSignatureUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/irker_worker.rb b/app/workers/irker_worker.rb index 4378da186a7..3097a9fbc03 100644 --- a/app/workers/irker_worker.rb +++ b/app/workers/irker_worker.rb @@ -6,9 +6,10 @@ require 'socket' class IrkerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - feature_category :integrations + urgency :low def perform(project_id, channels, colors, push_data, settings) # Establish connection to irker server diff --git a/app/workers/issuable/label_links_destroy_worker.rb b/app/workers/issuable/label_links_destroy_worker.rb index f663c410fba..f88c061bafb 100644 --- a/app/workers/issuable/label_links_destroy_worker.rb +++ b/app/workers/issuable/label_links_destroy_worker.rb @@ -4,6 +4,8 @@ module Issuable class LabelLinksDestroyWorker include ApplicationWorker + data_consistency :always + idempotent! feature_category :issue_tracking diff --git a/app/workers/issuable_export_csv_worker.rb b/app/workers/issuable_export_csv_worker.rb index 41facab6bb9..7e2c3407772 100644 --- a/app/workers/issuable_export_csv_worker.rb +++ b/app/workers/issuable_export_csv_worker.rb @@ -3,6 +3,8 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :issue_tracking diff --git a/app/workers/issuables/clear_groups_issue_counter_worker.rb b/app/workers/issuables/clear_groups_issue_counter_worker.rb index a8d6fd2f870..9e62224b83d 100644 --- a/app/workers/issuables/clear_groups_issue_counter_worker.rb +++ b/app/workers/issuables/clear_groups_issue_counter_worker.rb @@ -4,6 +4,8 @@ module Issuables class ClearGroupsIssueCounterWorker include ApplicationWorker + data_consistency :always + idempotent! urgency :low feature_category :issue_tracking diff --git a/app/workers/issue_due_scheduler_worker.rb b/app/workers/issue_due_scheduler_worker.rb index 9077b42d645..ad3470ae64b 100644 --- a/app/workers/issue_due_scheduler_worker.rb +++ b/app/workers/issue_due_scheduler_worker.rb @@ -3,7 +3,8 @@ class IssueDueSchedulerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :issue_tracking diff --git a/app/workers/issue_placement_worker.rb b/app/workers/issue_placement_worker.rb index 8166dda135e..e0c4502ed1a 100644 --- a/app/workers/issue_placement_worker.rb +++ b/app/workers/issue_placement_worker.rb @@ -3,6 +3,8 @@ class IssuePlacementWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 idempotent! diff --git a/app/workers/issue_rebalancing_worker.rb b/app/workers/issue_rebalancing_worker.rb index 66ef7dd3152..13e02c37bdb 100644 --- a/app/workers/issue_rebalancing_worker.rb +++ b/app/workers/issue_rebalancing_worker.rb @@ -3,6 +3,8 @@ class IssueRebalancingWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 idempotent! diff --git a/app/workers/jira_connect/forward_event_worker.rb b/app/workers/jira_connect/forward_event_worker.rb index 877ab46cfe5..14922db5420 100644 --- a/app/workers/jira_connect/forward_event_worker.rb +++ b/app/workers/jira_connect/forward_event_worker.rb @@ -4,22 +4,26 @@ module JiraConnect class ForwardEventWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always queue_namespace :jira_connect feature_category :integrations + urgency :low + worker_has_external_dependencies! def perform(installation_id, base_path, event_path) installation = JiraConnectInstallation.find_by_id(installation_id) + instance_url = installation&.instance_url + + installation.destroy if installation - return if installation&.instance_url.nil? + return if instance_url.nil? - proxy_url = installation.instance_url + event_path - qsh = Atlassian::Jwt.create_query_string_hash(proxy_url, 'POST', installation.instance_url + base_path) + proxy_url = instance_url + event_path + qsh = Atlassian::Jwt.create_query_string_hash(proxy_url, 'POST', instance_url + base_path) jwt = Atlassian::Jwt.encode({ iss: installation.client_key, qsh: qsh }, installation.shared_secret) - Gitlab::HTTP.post(proxy_url, headers: { 'Authorization' => "JWT #{jwt}" }) - ensure - installation.destroy if installation + JiraConnect::RetryRequestWorker.perform_async(proxy_url, jwt) end end end diff --git a/app/workers/jira_connect/retry_request_worker.rb b/app/workers/jira_connect/retry_request_worker.rb new file mode 100644 index 00000000000..b0f6dada639 --- /dev/null +++ b/app/workers/jira_connect/retry_request_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module JiraConnect + class RetryRequestWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + data_consistency :always + queue_namespace :jira_connect + feature_category :integrations + urgency :low + + worker_has_external_dependencies! + + def perform(proxy_url, jwt, attempts = 3) + r = Gitlab::HTTP.post(proxy_url, headers: { 'Authorization' => "JWT #{jwt}" }) + + self.class.perform_in(1.hour, proxy_url, jwt, attempts - 1) if r.code >= 400 && attempts > 0 + rescue *Gitlab::HTTP::HTTP_ERRORS + self.class.perform_in(1.hour, proxy_url, jwt, attempts - 1) if attempts > 0 + end + end +end diff --git a/app/workers/jira_connect/sync_branch_worker.rb b/app/workers/jira_connect/sync_branch_worker.rb index 2723287b77b..1b0655e4510 100644 --- a/app/workers/jira_connect/sync_branch_worker.rb +++ b/app/workers/jira_connect/sync_branch_worker.rb @@ -5,11 +5,11 @@ module JiraConnect include ApplicationWorker sidekiq_options retry: 3 - queue_namespace :jira_connect feature_category :integrations data_consistency :delayed loggable_arguments 1, 2 + urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb index 4c4daba3314..379f087bc1b 100644 --- a/app/workers/jira_connect/sync_builds_worker.rb +++ b/app/workers/jira_connect/sync_builds_worker.rb @@ -5,11 +5,11 @@ module JiraConnect include ApplicationWorker sidekiq_options retry: 3 - queue_namespace :jira_connect feature_category :integrations data_consistency :delayed tags :exclude_from_kubernetes + urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_deployments_worker.rb b/app/workers/jira_connect/sync_deployments_worker.rb index 0dc34b5999f..3138230ced5 100644 --- a/app/workers/jira_connect/sync_deployments_worker.rb +++ b/app/workers/jira_connect/sync_deployments_worker.rb @@ -5,11 +5,11 @@ module JiraConnect include ApplicationWorker sidekiq_options retry: 3 - queue_namespace :jira_connect feature_category :integrations data_consistency :delayed tags :exclude_from_kubernetes + urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_feature_flags_worker.rb b/app/workers/jira_connect/sync_feature_flags_worker.rb index c484cabbe6b..4de27c1b551 100644 --- a/app/workers/jira_connect/sync_feature_flags_worker.rb +++ b/app/workers/jira_connect/sync_feature_flags_worker.rb @@ -5,11 +5,11 @@ module JiraConnect include ApplicationWorker sidekiq_options retry: 3 - queue_namespace :jira_connect feature_category :integrations data_consistency :delayed tags :exclude_from_kubernetes + urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_merge_request_worker.rb b/app/workers/jira_connect/sync_merge_request_worker.rb index bb0d24667e9..6576aa9fdf4 100644 --- a/app/workers/jira_connect/sync_merge_request_worker.rb +++ b/app/workers/jira_connect/sync_merge_request_worker.rb @@ -5,10 +5,10 @@ module JiraConnect include ApplicationWorker sidekiq_options retry: 3 - queue_namespace :jira_connect feature_category :integrations data_consistency :delayed + urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_project_worker.rb b/app/workers/jira_connect/sync_project_worker.rb index 317bace89b4..a83444e58e7 100644 --- a/app/workers/jira_connect/sync_project_worker.rb +++ b/app/workers/jira_connect/sync_project_worker.rb @@ -5,11 +5,11 @@ module JiraConnect include ApplicationWorker sidekiq_options retry: 3 - queue_namespace :jira_connect feature_category :integrations data_consistency :delayed tags :exclude_from_kubernetes + urgency :low worker_has_external_dependencies! diff --git a/app/workers/mail_scheduler/issue_due_worker.rb b/app/workers/mail_scheduler/issue_due_worker.rb index 44f8f1e446c..4c17f8df722 100644 --- a/app/workers/mail_scheduler/issue_due_worker.rb +++ b/app/workers/mail_scheduler/issue_due_worker.rb @@ -4,6 +4,8 @@ module MailScheduler class IssueDueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include MailSchedulerQueue diff --git a/app/workers/mail_scheduler/notification_service_worker.rb b/app/workers/mail_scheduler/notification_service_worker.rb index 8645cc93511..0e6494a45d6 100644 --- a/app/workers/mail_scheduler/notification_service_worker.rb +++ b/app/workers/mail_scheduler/notification_service_worker.rb @@ -6,6 +6,8 @@ module MailScheduler class NotificationServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include MailSchedulerQueue diff --git a/app/workers/member_invitation_reminder_emails_worker.rb b/app/workers/member_invitation_reminder_emails_worker.rb index bfee8ab1fab..fa6787b9063 100644 --- a/app/workers/member_invitation_reminder_emails_worker.rb +++ b/app/workers/member_invitation_reminder_emails_worker.rb @@ -3,7 +3,8 @@ class MemberInvitationReminderEmailsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :subgroups diff --git a/app/workers/members_destroyer/unassign_issuables_worker.rb b/app/workers/members_destroyer/unassign_issuables_worker.rb index 0ee5654eaba..915551d6e30 100644 --- a/app/workers/members_destroyer/unassign_issuables_worker.rb +++ b/app/workers/members_destroyer/unassign_issuables_worker.rb @@ -4,6 +4,8 @@ module MembersDestroyer class UnassignIssuablesWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 ENTITY_TYPES = %w(Group Project).freeze diff --git a/app/workers/merge_request_cleanup_refs_worker.rb b/app/workers/merge_request_cleanup_refs_worker.rb index 408d070d56f..c57c6fbc28d 100644 --- a/app/workers/merge_request_cleanup_refs_worker.rb +++ b/app/workers/merge_request_cleanup_refs_worker.rb @@ -5,6 +5,8 @@ class MergeRequestCleanupRefsWorker include LimitedCapacity::Worker include Gitlab::Utils::StrongMemoize + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review diff --git a/app/workers/merge_request_mergeability_check_worker.rb b/app/workers/merge_request_mergeability_check_worker.rb index 13961de1f59..0e1ab505644 100644 --- a/app/workers/merge_request_mergeability_check_worker.rb +++ b/app/workers/merge_request_mergeability_check_worker.rb @@ -3,16 +3,22 @@ class MergeRequestMergeabilityCheckWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review idempotent! + def logger + @logger ||= Sidekiq.logger + end + def perform(merge_request_id) merge_request = MergeRequest.find_by_id(merge_request_id) unless merge_request - logger.error("Failed to find merge request with ID: #{merge_request_id}") + logger.error(worker: self.class.name, message: "Failed to find merge request", merge_request_id: merge_request_id) return end @@ -21,6 +27,6 @@ class MergeRequestMergeabilityCheckWorker .new(merge_request) .execute(recheck: false, retry_lease: false) - logger.error("Failed to check mergeability of merge request (#{merge_request_id}): #{result.message}") if result.error? + logger.error(worker: self.class.name, message: "Failed to check mergeability of merge request: #{result.message}", merge_request_id: merge_request_id) if result.error? end end diff --git a/app/workers/merge_requests/create_pipeline_worker.rb b/app/workers/merge_requests/create_pipeline_worker.rb index a79a92a5419..ee42a3dee08 100644 --- a/app/workers/merge_requests/create_pipeline_worker.rb +++ b/app/workers/merge_requests/create_pipeline_worker.rb @@ -4,6 +4,8 @@ module MergeRequests class CreatePipelineWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/merge_requests/delete_source_branch_worker.rb b/app/workers/merge_requests/delete_source_branch_worker.rb index 1ce3a99b298..69bd3949e9d 100644 --- a/app/workers/merge_requests/delete_source_branch_worker.rb +++ b/app/workers/merge_requests/delete_source_branch_worker.rb @@ -3,6 +3,8 @@ class MergeRequests::DeleteSourceBranchWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/merge_requests/handle_assignees_change_worker.rb b/app/workers/merge_requests/handle_assignees_change_worker.rb index 4c0500cd520..7cf1be51d23 100644 --- a/app/workers/merge_requests/handle_assignees_change_worker.rb +++ b/app/workers/merge_requests/handle_assignees_change_worker.rb @@ -3,6 +3,8 @@ class MergeRequests::HandleAssigneesChangeWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review diff --git a/app/workers/merge_requests/resolve_todos_worker.rb b/app/workers/merge_requests/resolve_todos_worker.rb index 8bb88091efe..314cdac4414 100644 --- a/app/workers/merge_requests/resolve_todos_worker.rb +++ b/app/workers/merge_requests/resolve_todos_worker.rb @@ -3,6 +3,8 @@ class MergeRequests::ResolveTodosWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index df5a7a904fc..3fcd7a3ad7a 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -3,6 +3,8 @@ class MergeWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/metrics/dashboard/prune_old_annotations_worker.rb b/app/workers/metrics/dashboard/prune_old_annotations_worker.rb index 5e8067a4438..5c117486da2 100644 --- a/app/workers/metrics/dashboard/prune_old_annotations_worker.rb +++ b/app/workers/metrics/dashboard/prune_old_annotations_worker.rb @@ -5,6 +5,8 @@ module Metrics class PruneOldAnnotationsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 DELETE_LIMIT = 10_000 diff --git a/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb b/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb index 6f2ff8cca13..62cf35a669f 100644 --- a/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb +++ b/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb @@ -5,7 +5,8 @@ module Metrics class ScheduleAnnotationsPruneWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/metrics/dashboard/sync_dashboards_worker.rb b/app/workers/metrics/dashboard/sync_dashboards_worker.rb index 0fdc7825f47..645c03428a2 100644 --- a/app/workers/metrics/dashboard/sync_dashboards_worker.rb +++ b/app/workers/metrics/dashboard/sync_dashboards_worker.rb @@ -5,6 +5,8 @@ module Metrics class SyncDashboardsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :metrics diff --git a/app/workers/migrate_external_diffs_worker.rb b/app/workers/migrate_external_diffs_worker.rb index a73a9be4f0c..566797d8b8a 100644 --- a/app/workers/migrate_external_diffs_worker.rb +++ b/app/workers/migrate_external_diffs_worker.rb @@ -3,6 +3,8 @@ class MigrateExternalDiffsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review diff --git a/app/workers/namespaceless_project_destroy_worker.rb b/app/workers/namespaceless_project_destroy_worker.rb index 91cad6f2a5c..c2ed379be48 100644 --- a/app/workers/namespaceless_project_destroy_worker.rb +++ b/app/workers/namespaceless_project_destroy_worker.rb @@ -9,6 +9,8 @@ class NamespacelessProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExceptionBacktrace diff --git a/app/workers/namespaces/in_product_marketing_emails_worker.rb b/app/workers/namespaces/in_product_marketing_emails_worker.rb index 1f46be29553..035fa453f59 100644 --- a/app/workers/namespaces/in_product_marketing_emails_worker.rb +++ b/app/workers/namespaces/in_product_marketing_emails_worker.rb @@ -4,7 +4,8 @@ module Namespaces class InProductMarketingEmailsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :subgroups diff --git a/app/workers/namespaces/onboarding_issue_created_worker.rb b/app/workers/namespaces/onboarding_issue_created_worker.rb index 7b8b1a43078..3cff741ecbf 100644 --- a/app/workers/namespaces/onboarding_issue_created_worker.rb +++ b/app/workers/namespaces/onboarding_issue_created_worker.rb @@ -4,6 +4,8 @@ module Namespaces class OnboardingIssueCreatedWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :issue_tracking diff --git a/app/workers/namespaces/onboarding_pipeline_created_worker.rb b/app/workers/namespaces/onboarding_pipeline_created_worker.rb index 128d7b6aa06..2c77fab8114 100644 --- a/app/workers/namespaces/onboarding_pipeline_created_worker.rb +++ b/app/workers/namespaces/onboarding_pipeline_created_worker.rb @@ -4,6 +4,8 @@ module Namespaces class OnboardingPipelineCreatedWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :subgroups diff --git a/app/workers/namespaces/onboarding_progress_worker.rb b/app/workers/namespaces/onboarding_progress_worker.rb index d4db55a9207..43d13618091 100644 --- a/app/workers/namespaces/onboarding_progress_worker.rb +++ b/app/workers/namespaces/onboarding_progress_worker.rb @@ -4,9 +4,12 @@ module Namespaces class OnboardingProgressWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :product_analytics + worker_resource_boundary :cpu tags :exclude_from_kubernetes urgency :low diff --git a/app/workers/namespaces/onboarding_user_added_worker.rb b/app/workers/namespaces/onboarding_user_added_worker.rb index 8c85dfafa12..4d4d9c03d3e 100644 --- a/app/workers/namespaces/onboarding_user_added_worker.rb +++ b/app/workers/namespaces/onboarding_user_added_worker.rb @@ -4,6 +4,8 @@ module Namespaces class OnboardingUserAddedWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :users diff --git a/app/workers/namespaces/prune_aggregation_schedules_worker.rb b/app/workers/namespaces/prune_aggregation_schedules_worker.rb index 0ea27c532ae..1084edb9b50 100644 --- a/app/workers/namespaces/prune_aggregation_schedules_worker.rb +++ b/app/workers/namespaces/prune_aggregation_schedules_worker.rb @@ -4,7 +4,8 @@ module Namespaces class PruneAggregationSchedulesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :source_code_management diff --git a/app/workers/namespaces/root_statistics_worker.rb b/app/workers/namespaces/root_statistics_worker.rb index 92bf2e22020..b97dbca2c1c 100644 --- a/app/workers/namespaces/root_statistics_worker.rb +++ b/app/workers/namespaces/root_statistics_worker.rb @@ -4,6 +4,8 @@ module Namespaces class RootStatisticsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :update_namespace_statistics diff --git a/app/workers/namespaces/schedule_aggregation_worker.rb b/app/workers/namespaces/schedule_aggregation_worker.rb index cee273688e9..7cd7f5223d6 100644 --- a/app/workers/namespaces/schedule_aggregation_worker.rb +++ b/app/workers/namespaces/schedule_aggregation_worker.rb @@ -4,6 +4,8 @@ module Namespaces class ScheduleAggregationWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :update_namespace_statistics diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb index a579b828354..899545fc02c 100644 --- a/app/workers/new_issue_worker.rb +++ b/app/workers/new_issue_worker.rb @@ -3,6 +3,8 @@ class NewIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include NewIssuable diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb index 574c73ad3b5..9694d44e8c1 100644 --- a/app/workers/new_merge_request_worker.rb +++ b/app/workers/new_merge_request_worker.rb @@ -3,6 +3,8 @@ class NewMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include NewIssuable diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index 566bb9a9057..e54d84e382e 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -3,6 +3,8 @@ class NewNoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :issue_tracking diff --git a/app/workers/object_pool/create_worker.rb b/app/workers/object_pool/create_worker.rb index 586b81fcd30..28e130ee2bc 100644 --- a/app/workers/object_pool/create_worker.rb +++ b/app/workers/object_pool/create_worker.rb @@ -4,6 +4,8 @@ module ObjectPool class CreateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ObjectPoolQueue include ExclusiveLeaseGuard diff --git a/app/workers/object_pool/destroy_worker.rb b/app/workers/object_pool/destroy_worker.rb index 297780b20bd..db28c5ba064 100644 --- a/app/workers/object_pool/destroy_worker.rb +++ b/app/workers/object_pool/destroy_worker.rb @@ -4,6 +4,8 @@ module ObjectPool class DestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ObjectPoolQueue diff --git a/app/workers/object_pool/join_worker.rb b/app/workers/object_pool/join_worker.rb index 282a8f54695..36094ddbe84 100644 --- a/app/workers/object_pool/join_worker.rb +++ b/app/workers/object_pool/join_worker.rb @@ -4,6 +4,8 @@ module ObjectPool class JoinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ObjectPoolQueue diff --git a/app/workers/object_pool/schedule_join_worker.rb b/app/workers/object_pool/schedule_join_worker.rb index 44208208d04..a2f4ba63b69 100644 --- a/app/workers/object_pool/schedule_join_worker.rb +++ b/app/workers/object_pool/schedule_join_worker.rb @@ -4,6 +4,8 @@ module ObjectPool class ScheduleJoinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ObjectPoolQueue diff --git a/app/workers/object_storage/background_move_worker.rb b/app/workers/object_storage/background_move_worker.rb index fba91e49e43..2204e504702 100644 --- a/app/workers/object_storage/background_move_worker.rb +++ b/app/workers/object_storage/background_move_worker.rb @@ -3,6 +3,8 @@ module ObjectStorage class BackgroundMoveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + data_consistency :always include ObjectStorageQueue sidekiq_options retry: 5 diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb index 7323ab50370..ea4a90cf9d2 100644 --- a/app/workers/object_storage/migrate_uploads_worker.rb +++ b/app/workers/object_storage/migrate_uploads_worker.rb @@ -5,6 +5,8 @@ module ObjectStorage class MigrateUploadsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ObjectStorageQueue diff --git a/app/workers/packages/composer/cache_cleanup_worker.rb b/app/workers/packages/composer/cache_cleanup_worker.rb index 1d47ef87962..fee886bc750 100644 --- a/app/workers/packages/composer/cache_cleanup_worker.rb +++ b/app/workers/packages/composer/cache_cleanup_worker.rb @@ -5,7 +5,8 @@ module Packages class CacheCleanupWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :package_registry diff --git a/app/workers/packages/composer/cache_update_worker.rb b/app/workers/packages/composer/cache_update_worker.rb index d87abf2e256..f146a0a83cc 100644 --- a/app/workers/packages/composer/cache_update_worker.rb +++ b/app/workers/packages/composer/cache_update_worker.rb @@ -5,6 +5,8 @@ module Packages class CacheUpdateWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :package_registry diff --git a/app/workers/packages/debian/generate_distribution_worker.rb b/app/workers/packages/debian/generate_distribution_worker.rb index 68fdd80ffb1..b9b157d25d2 100644 --- a/app/workers/packages/debian/generate_distribution_worker.rb +++ b/app/workers/packages/debian/generate_distribution_worker.rb @@ -4,6 +4,8 @@ module Packages module Debian class GenerateDistributionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + data_consistency :always include Gitlab::Utils::StrongMemoize # The worker is idempotent, by reusing component files with the same file_sha256. diff --git a/app/workers/packages/debian/process_changes_worker.rb b/app/workers/packages/debian/process_changes_worker.rb index edc366a7597..3ddeb858429 100644 --- a/app/workers/packages/debian/process_changes_worker.rb +++ b/app/workers/packages/debian/process_changes_worker.rb @@ -4,6 +4,8 @@ module Packages module Debian class ProcessChangesWorker include ApplicationWorker + + data_consistency :always include Gitlab::Utils::StrongMemoize deduplicate :until_executed diff --git a/app/workers/packages/go/sync_packages_worker.rb b/app/workers/packages/go/sync_packages_worker.rb index c5f631c47db..182c9bfec5d 100644 --- a/app/workers/packages/go/sync_packages_worker.rb +++ b/app/workers/packages/go/sync_packages_worker.rb @@ -5,6 +5,8 @@ module Packages class SyncPackagesWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include Gitlab::Golang diff --git a/app/workers/packages/helm/extraction_worker.rb b/app/workers/packages/helm/extraction_worker.rb index fd4e720da94..1010a0833b1 100644 --- a/app/workers/packages/helm/extraction_worker.rb +++ b/app/workers/packages/helm/extraction_worker.rb @@ -5,6 +5,8 @@ module Packages class ExtractionWorker include ApplicationWorker + data_consistency :always + queue_namespace :package_repositories feature_category :package_registry deduplicate :until_executing diff --git a/app/workers/packages/maven/metadata/sync_worker.rb b/app/workers/packages/maven/metadata/sync_worker.rb index c53117a08c5..ab18c70e95e 100644 --- a/app/workers/packages/maven/metadata/sync_worker.rb +++ b/app/workers/packages/maven/metadata/sync_worker.rb @@ -6,6 +6,8 @@ module Packages class SyncWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include Gitlab::Utils::StrongMemoize diff --git a/app/workers/packages/nuget/extraction_worker.rb b/app/workers/packages/nuget/extraction_worker.rb index 4128b229ebe..b8e00621aa1 100644 --- a/app/workers/packages/nuget/extraction_worker.rb +++ b/app/workers/packages/nuget/extraction_worker.rb @@ -5,6 +5,8 @@ module Packages class ExtractionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :package_repositories diff --git a/app/workers/packages/rubygems/extraction_worker.rb b/app/workers/packages/rubygems/extraction_worker.rb index fc32654a2c1..520305981cf 100644 --- a/app/workers/packages/rubygems/extraction_worker.rb +++ b/app/workers/packages/rubygems/extraction_worker.rb @@ -5,6 +5,8 @@ module Packages class ExtractionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 queue_namespace :package_repositories diff --git a/app/workers/pages_domain_removal_cron_worker.rb b/app/workers/pages_domain_removal_cron_worker.rb index cc720676214..4744c855394 100644 --- a/app/workers/pages_domain_removal_cron_worker.rb +++ b/app/workers/pages_domain_removal_cron_worker.rb @@ -3,7 +3,8 @@ class PagesDomainRemovalCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :pages diff --git a/app/workers/pages_domain_ssl_renewal_cron_worker.rb b/app/workers/pages_domain_ssl_renewal_cron_worker.rb index c99eed8a8df..a551078c91a 100644 --- a/app/workers/pages_domain_ssl_renewal_cron_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_cron_worker.rb @@ -3,10 +3,12 @@ class PagesDomainSslRenewalCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :pages + worker_resource_boundary :cpu def perform return unless ::Gitlab::LetsEncrypt.enabled? diff --git a/app/workers/pages_domain_ssl_renewal_worker.rb b/app/workers/pages_domain_ssl_renewal_worker.rb index 2ab41aab795..d4c68f66699 100644 --- a/app/workers/pages_domain_ssl_renewal_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_worker.rb @@ -3,6 +3,8 @@ class PagesDomainSslRenewalWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :pages diff --git a/app/workers/pages_domain_verification_cron_worker.rb b/app/workers/pages_domain_verification_cron_worker.rb index ec63004716a..56339d50a40 100644 --- a/app/workers/pages_domain_verification_cron_worker.rb +++ b/app/workers/pages_domain_verification_cron_worker.rb @@ -3,10 +3,12 @@ class PagesDomainVerificationCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :pages + worker_resource_boundary :cpu def perform return if Gitlab::Database.read_only? diff --git a/app/workers/pages_domain_verification_worker.rb b/app/workers/pages_domain_verification_worker.rb index b67b1b4d8ee..f9504a7c1d2 100644 --- a/app/workers/pages_domain_verification_worker.rb +++ b/app/workers/pages_domain_verification_worker.rb @@ -3,6 +3,8 @@ class PagesDomainVerificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :pages diff --git a/app/workers/pages_remove_worker.rb b/app/workers/pages_remove_worker.rb index 3e60df9027a..69a8344b5aa 100644 --- a/app/workers/pages_remove_worker.rb +++ b/app/workers/pages_remove_worker.rb @@ -4,6 +4,8 @@ class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :pages tags :exclude_from_kubernetes diff --git a/app/workers/pages_transfer_worker.rb b/app/workers/pages_transfer_worker.rb index 0d80ec28310..c2190a352dd 100644 --- a/app/workers/pages_transfer_worker.rb +++ b/app/workers/pages_transfer_worker.rb @@ -3,6 +3,8 @@ class PagesTransferWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 TransferFailedError = Class.new(StandardError) diff --git a/app/workers/pages_update_configuration_worker.rb b/app/workers/pages_update_configuration_worker.rb index 8bb9f76670b..8a37b70a0b7 100644 --- a/app/workers/pages_update_configuration_worker.rb +++ b/app/workers/pages_update_configuration_worker.rb @@ -3,6 +3,8 @@ class PagesUpdateConfigurationWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 idempotent! diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index ee394271653..d0c21cf74e1 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -3,10 +3,13 @@ class PagesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :pages loggable_arguments 0, 1 tags :requires_disk_io, :exclude_from_kubernetes + worker_resource_boundary :cpu def perform(action, *arg) send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/workers/partition_creation_worker.rb b/app/workers/partition_creation_worker.rb index bb4834ab2dd..3f74dc3812a 100644 --- a/app/workers/partition_creation_worker.rb +++ b/app/workers/partition_creation_worker.rb @@ -3,7 +3,8 @@ class PartitionCreationWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :database diff --git a/app/workers/personal_access_tokens/expired_notification_worker.rb b/app/workers/personal_access_tokens/expired_notification_worker.rb index 73568960d38..8deacf457b2 100644 --- a/app/workers/personal_access_tokens/expired_notification_worker.rb +++ b/app/workers/personal_access_tokens/expired_notification_worker.rb @@ -4,7 +4,8 @@ module PersonalAccessTokens class ExpiredNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :authentication_and_authorization diff --git a/app/workers/personal_access_tokens/expiring_worker.rb b/app/workers/personal_access_tokens/expiring_worker.rb index aaca78e3c63..f4afa9f8994 100644 --- a/app/workers/personal_access_tokens/expiring_worker.rb +++ b/app/workers/personal_access_tokens/expiring_worker.rb @@ -4,7 +4,8 @@ module PersonalAccessTokens class ExpiringWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :authentication_and_authorization diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb index 40d138752b4..322f92d376b 100644 --- a/app/workers/pipeline_hooks_worker.rb +++ b/app/workers/pipeline_hooks_worker.rb @@ -12,7 +12,7 @@ class PipelineHooksWorker # rubocop:disable Scalability/IdempotentWorker # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) - Ci::Pipeline.includes({ builds: { runner: :tags } }) + Ci::Pipeline .find_by(id: pipeline_id) .try(:execute_hooks) end diff --git a/app/workers/pipeline_metrics_worker.rb b/app/workers/pipeline_metrics_worker.rb index fdab10d7dda..c2580c3d48b 100644 --- a/app/workers/pipeline_metrics_worker.rb +++ b/app/workers/pipeline_metrics_worker.rb @@ -3,6 +3,8 @@ class PipelineMetricsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/pipeline_notification_worker.rb b/app/workers/pipeline_notification_worker.rb index 619570dcf41..640f3494d58 100644 --- a/app/workers/pipeline_notification_worker.rb +++ b/app/workers/pipeline_notification_worker.rb @@ -8,6 +8,7 @@ class PipelineNotificationWorker # rubocop:disable Scalability/IdempotentWorker urgency :high worker_resource_boundary :cpu + data_consistency :delayed def perform(pipeline_id, args = {}) case args diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb index a35b32c35f2..9cd471a5ab6 100644 --- a/app/workers/pipeline_process_worker.rb +++ b/app/workers/pipeline_process_worker.rb @@ -3,6 +3,8 @@ class PipelineProcessWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb index f1248ec9e58..ebda30f57d8 100644 --- a/app/workers/pipeline_schedule_worker.rb +++ b/app/workers/pipeline_schedule_worker.rb @@ -3,7 +3,8 @@ class PipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :continuous_integration diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index ce985492935..4a49e18eb9b 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -3,6 +3,8 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include Gitlab::Experiment::Dsl @@ -124,7 +126,6 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker end def after_project_changes_hooks(project, user, refs, changes) - experiment(:new_project_readme, actor: user).track_initial_writes(project) experiment(:empty_repo_upload, project: project).track_initial_write repository_update_hook_data = Gitlab::DataBuilder::Repository.update(project, user, changes, refs) SystemHooksService.new.execute_hooks(repository_update_hook_data, :repository_update_hooks) diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index 54ffe8d3b10..a4dfe11c394 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -10,6 +10,8 @@ class ProcessCommitWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index d2796cdb697..328fdc4717c 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -4,6 +4,8 @@ class ProjectCacheWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 LEASE_TIMEOUT = 15.minutes.to_i diff --git a/app/workers/project_daily_statistics_worker.rb b/app/workers/project_daily_statistics_worker.rb index 7d673ec00d9..02f8958f82a 100644 --- a/app/workers/project_daily_statistics_worker.rb +++ b/app/workers/project_daily_statistics_worker.rb @@ -4,6 +4,8 @@ class ProjectDailyStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb index be11fa65028..149f8290b54 100644 --- a/app/workers/project_destroy_worker.rb +++ b/app/workers/project_destroy_worker.rb @@ -3,6 +3,8 @@ class ProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExceptionBacktrace diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 1c4aa3f7e49..4dd9a9c6fcb 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -2,6 +2,8 @@ class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + data_consistency :always include ExceptionBacktrace feature_category :importers diff --git a/app/workers/project_service_worker.rb b/app/workers/project_service_worker.rb index da38d2fc0cd..6d809dfb22b 100644 --- a/app/workers/project_service_worker.rb +++ b/app/workers/project_service_worker.rb @@ -3,10 +3,12 @@ class ProjectServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - sidekiq_options dead: false feature_category :integrations + urgency :low + worker_has_external_dependencies! def perform(hook_id, data) diff --git a/app/workers/projects/post_creation_worker.rb b/app/workers/projects/post_creation_worker.rb index 389e987e81a..99438e4e4b2 100644 --- a/app/workers/projects/post_creation_worker.rb +++ b/app/workers/projects/post_creation_worker.rb @@ -4,6 +4,8 @@ module Projects class PostCreationWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb b/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb index 55530bff7c1..d7f207f45e8 100644 --- a/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb +++ b/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb @@ -4,6 +4,8 @@ module Projects class ScheduleBulkRepositoryShardMovesWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 idempotent! diff --git a/app/workers/prometheus/create_default_alerts_worker.rb b/app/workers/prometheus/create_default_alerts_worker.rb index 9d163cd828e..94ac02c4c04 100644 --- a/app/workers/prometheus/create_default_alerts_worker.rb +++ b/app/workers/prometheus/create_default_alerts_worker.rb @@ -4,6 +4,8 @@ module Prometheus class CreateDefaultAlertsWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :incident_management diff --git a/app/workers/propagate_integration_group_worker.rb b/app/workers/propagate_integration_group_worker.rb index 6881740461f..443ff1f2abe 100644 --- a/app/workers/propagate_integration_group_worker.rb +++ b/app/workers/propagate_integration_group_worker.rb @@ -3,10 +3,12 @@ class PropagateIntegrationGroupWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - feature_category :integrations tags :exclude_from_kubernetes + urgency :low + idempotent! # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/propagate_integration_inherit_descendant_worker.rb b/app/workers/propagate_integration_inherit_descendant_worker.rb index 9067af12de3..24573591409 100644 --- a/app/workers/propagate_integration_inherit_descendant_worker.rb +++ b/app/workers/propagate_integration_inherit_descendant_worker.rb @@ -3,10 +3,12 @@ class PropagateIntegrationInheritDescendantWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - feature_category :integrations tags :exclude_from_kubernetes + urgency :low + idempotent! # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/propagate_integration_inherit_worker.rb b/app/workers/propagate_integration_inherit_worker.rb index e7649d6714f..24a8778b928 100644 --- a/app/workers/propagate_integration_inherit_worker.rb +++ b/app/workers/propagate_integration_inherit_worker.rb @@ -3,10 +3,12 @@ class PropagateIntegrationInheritWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - feature_category :integrations tags :exclude_from_kubernetes + urgency :low + idempotent! # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/propagate_integration_project_worker.rb b/app/workers/propagate_integration_project_worker.rb index 90cf27c4176..dba8a270007 100644 --- a/app/workers/propagate_integration_project_worker.rb +++ b/app/workers/propagate_integration_project_worker.rb @@ -3,10 +3,12 @@ class PropagateIntegrationProjectWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - feature_category :integrations tags :exclude_from_kubernetes + urgency :low + idempotent! # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/propagate_integration_worker.rb b/app/workers/propagate_integration_worker.rb index 0f8229bdf09..9d21d92b6e3 100644 --- a/app/workers/propagate_integration_worker.rb +++ b/app/workers/propagate_integration_worker.rb @@ -3,11 +3,13 @@ class PropagateIntegrationWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - feature_category :integrations - idempotent! loggable_arguments 1 + urgency :low + + idempotent! def perform(integration_id) Admin::PropagateIntegrationService.propagate(Integration.find(integration_id)) diff --git a/app/workers/propagate_service_template_worker.rb b/app/workers/propagate_service_template_worker.rb index 149577b15cd..6b8382ae433 100644 --- a/app/workers/propagate_service_template_worker.rb +++ b/app/workers/propagate_service_template_worker.rb @@ -1,9 +1,12 @@ # frozen_string_literal: true -# Worker for updating any project specific caches. +# No longer in use https://gitlab.com/groups/gitlab-org/-/epics/5672 +# To be removed https://gitlab.com/gitlab-org/gitlab/-/issues/335178 class PropagateServiceTemplateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :integrations diff --git a/app/workers/prune_old_events_worker.rb b/app/workers/prune_old_events_worker.rb index 59d324bc573..2d74c271ddc 100644 --- a/app/workers/prune_old_events_worker.rb +++ b/app/workers/prune_old_events_worker.rb @@ -3,7 +3,8 @@ class PruneOldEventsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/purge_dependency_proxy_cache_worker.rb b/app/workers/purge_dependency_proxy_cache_worker.rb index 9f1ea8a6eb4..8ab4e77bc78 100644 --- a/app/workers/purge_dependency_proxy_cache_worker.rb +++ b/app/workers/purge_dependency_proxy_cache_worker.rb @@ -3,6 +3,8 @@ class PurgeDependencyProxyCacheWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include Gitlab::Allowable idempotent! diff --git a/app/workers/rebase_worker.rb b/app/workers/rebase_worker.rb index 664905eb9e5..7121ce0e205 100644 --- a/app/workers/rebase_worker.rb +++ b/app/workers/rebase_worker.rb @@ -5,6 +5,8 @@ class RebaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/releases/create_evidence_worker.rb b/app/workers/releases/create_evidence_worker.rb index bd790e8d0ee..5aed543500f 100644 --- a/app/workers/releases/create_evidence_worker.rb +++ b/app/workers/releases/create_evidence_worker.rb @@ -4,6 +4,8 @@ module Releases class CreateEvidenceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :release_evidence diff --git a/app/workers/releases/manage_evidence_worker.rb b/app/workers/releases/manage_evidence_worker.rb index 88b6c4aea06..f316aa6eefd 100644 --- a/app/workers/releases/manage_evidence_worker.rb +++ b/app/workers/releases/manage_evidence_worker.rb @@ -4,7 +4,8 @@ module Releases class ManageEvidenceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :release_evidence diff --git a/app/workers/remote_mirror_notification_worker.rb b/app/workers/remote_mirror_notification_worker.rb index 39a7c0fc79d..ed3a0ccb989 100644 --- a/app/workers/remote_mirror_notification_worker.rb +++ b/app/workers/remote_mirror_notification_worker.rb @@ -3,6 +3,8 @@ class RemoteMirrorNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/remove_expired_group_links_worker.rb b/app/workers/remove_expired_group_links_worker.rb index edf3a02cff5..37298c53a5c 100644 --- a/app/workers/remove_expired_group_links_worker.rb +++ b/app/workers/remove_expired_group_links_worker.rb @@ -3,7 +3,8 @@ class RemoveExpiredGroupLinksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :authentication_and_authorization diff --git a/app/workers/remove_expired_members_worker.rb b/app/workers/remove_expired_members_worker.rb index 9940953207e..c9eb715a522 100644 --- a/app/workers/remove_expired_members_worker.rb +++ b/app/workers/remove_expired_members_worker.rb @@ -3,7 +3,8 @@ class RemoveExpiredMembersWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :authentication_and_authorization diff --git a/app/workers/remove_unaccepted_member_invites_worker.rb b/app/workers/remove_unaccepted_member_invites_worker.rb index c1f8e3881f1..7833ec30c3c 100644 --- a/app/workers/remove_unaccepted_member_invites_worker.rb +++ b/app/workers/remove_unaccepted_member_invites_worker.rb @@ -3,7 +3,8 @@ class RemoveUnacceptedMemberInvitesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :authentication_and_authorization diff --git a/app/workers/remove_unreferenced_lfs_objects_worker.rb b/app/workers/remove_unreferenced_lfs_objects_worker.rb index ca4b70a0485..c67ab6e356a 100644 --- a/app/workers/remove_unreferenced_lfs_objects_worker.rb +++ b/app/workers/remove_unreferenced_lfs_objects_worker.rb @@ -3,7 +3,8 @@ class RemoveUnreferencedLfsObjectsWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/repository_archive_cache_worker.rb b/app/workers/repository_archive_cache_worker.rb index 84cafba17cf..c713f54faa5 100644 --- a/app/workers/repository_archive_cache_worker.rb +++ b/app/workers/repository_archive_cache_worker.rb @@ -3,7 +3,8 @@ class RepositoryArchiveCacheWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index a8744638d7b..88a8f98aaf6 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -4,6 +4,8 @@ module RepositoryCheck class BatchWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include RepositoryCheckQueue include ExclusiveLeaseGuard diff --git a/app/workers/repository_check/clear_worker.rb b/app/workers/repository_check/clear_worker.rb index bc19b42da1a..5d2db1b4472 100644 --- a/app/workers/repository_check/clear_worker.rb +++ b/app/workers/repository_check/clear_worker.rb @@ -4,6 +4,8 @@ module RepositoryCheck class ClearWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include RepositoryCheckQueue diff --git a/app/workers/repository_check/dispatch_worker.rb b/app/workers/repository_check/dispatch_worker.rb index 30734926765..fc41eab2c16 100644 --- a/app/workers/repository_check/dispatch_worker.rb +++ b/app/workers/repository_check/dispatch_worker.rb @@ -4,7 +4,8 @@ module RepositoryCheck class DispatchWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb index 31d68e65b23..4c74a647464 100644 --- a/app/workers/repository_check/single_repository_worker.rb +++ b/app/workers/repository_check/single_repository_worker.rb @@ -4,6 +4,8 @@ module RepositoryCheck class SingleRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include RepositoryCheckQueue diff --git a/app/workers/repository_cleanup_worker.rb b/app/workers/repository_cleanup_worker.rb index 03c9add6afb..2b4f41f9612 100644 --- a/app/workers/repository_cleanup_worker.rb +++ b/app/workers/repository_cleanup_worker.rb @@ -3,6 +3,8 @@ class RepositoryCleanupWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index 06a6f5b0600..5ec9ceaf004 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -3,6 +3,8 @@ class RepositoryForkWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ProjectStartImport include ProjectImportOptions diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 0f86d55df22..413bb135943 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -2,6 +2,8 @@ class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + data_consistency :always include ExceptionBacktrace include ProjectStartImport diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb index 48158cda857..c95393e7d21 100644 --- a/app/workers/repository_remove_remote_worker.rb +++ b/app/workers/repository_remove_remote_worker.rb @@ -3,6 +3,8 @@ class RepositoryRemoveRemoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExclusiveLeaseGuard @@ -14,22 +16,13 @@ class RepositoryRemoveRemoteWorker # rubocop:disable Scalability/IdempotentWorke attr_reader :project, :remote_name def perform(project_id, remote_name) - @remote_name = remote_name - @project = Project.find_by_id(project_id) - - return unless @project - - logger.info("Removing remote #{remote_name} from project #{project.id}") - - try_obtain_lease do - remove_remote = @project.repository.remove_remote(remote_name) - - if remove_remote - logger.info("Remote #{remote_name} was successfully removed from project #{project.id}") - else - logger.error("Could not remove remote #{remote_name} from project #{project.id}") - end - end + # On-disk remotes are slated for removal, and GitLab doesn't create any of + # them anymore. For backwards compatibility, we need to keep the worker + # though such that we can be sure to drain all jobs on an update. Making + # this a no-op is fine though: the worst that can happen is that we still + # have old remotes lingering in the repository's config, but Gitaly will + # start to clean these up in repository maintenance. + # https://gitlab.com/gitlab-org/gitlab/-/issues/336745 end def lease_timeout diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb index 483aae84a3b..9265449fdf4 100644 --- a/app/workers/repository_update_remote_mirror_worker.rb +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -4,6 +4,8 @@ class RepositoryUpdateRemoteMirrorWorker UpdateError = Class.new(StandardError) include ApplicationWorker + + data_consistency :always include Gitlab::ExclusiveLeaseHelpers worker_has_external_dependencies! diff --git a/app/workers/requests_profiles_worker.rb b/app/workers/requests_profiles_worker.rb index 35c18177a81..e02b63fb621 100644 --- a/app/workers/requests_profiles_worker.rb +++ b/app/workers/requests_profiles_worker.rb @@ -3,7 +3,8 @@ class RequestsProfilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb index 553153848c7..dd0f14a5cab 100644 --- a/app/workers/run_pipeline_schedule_worker.rb +++ b/app/workers/run_pipeline_schedule_worker.rb @@ -3,6 +3,8 @@ class RunPipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/schedule_merge_request_cleanup_refs_worker.rb b/app/workers/schedule_merge_request_cleanup_refs_worker.rb index 40a773ca58f..46a6e0ef01f 100644 --- a/app/workers/schedule_merge_request_cleanup_refs_worker.rb +++ b/app/workers/schedule_merge_request_cleanup_refs_worker.rb @@ -3,7 +3,8 @@ class ScheduleMergeRequestCleanupRefsWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :code_review diff --git a/app/workers/schedule_migrate_external_diffs_worker.rb b/app/workers/schedule_migrate_external_diffs_worker.rb index ecafe8f5e7d..b2dea5083b4 100644 --- a/app/workers/schedule_migrate_external_diffs_worker.rb +++ b/app/workers/schedule_migrate_external_diffs_worker.rb @@ -3,7 +3,8 @@ class ScheduleMigrateExternalDiffsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext: # This schedules the `MigrateExternalDiffsWorker` # issue for adding context: https://gitlab.com/gitlab-org/gitlab/issues/202100 diff --git a/app/workers/self_monitoring_project_create_worker.rb b/app/workers/self_monitoring_project_create_worker.rb index 9dc3bb855fb..97d858eddd9 100644 --- a/app/workers/self_monitoring_project_create_worker.rb +++ b/app/workers/self_monitoring_project_create_worker.rb @@ -3,6 +3,8 @@ class SelfMonitoringProjectCreateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExclusiveLeaseGuard include SelfMonitoringProjectWorker diff --git a/app/workers/self_monitoring_project_delete_worker.rb b/app/workers/self_monitoring_project_delete_worker.rb index c155c57dec7..74dc4cb6581 100644 --- a/app/workers/self_monitoring_project_delete_worker.rb +++ b/app/workers/self_monitoring_project_delete_worker.rb @@ -3,6 +3,8 @@ class SelfMonitoringProjectDeleteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ExclusiveLeaseGuard include SelfMonitoringProjectWorker diff --git a/app/workers/service_desk_email_receiver_worker.rb b/app/workers/service_desk_email_receiver_worker.rb index cfe63e059bb..f546fce3e8a 100644 --- a/app/workers/service_desk_email_receiver_worker.rb +++ b/app/workers/service_desk_email_receiver_worker.rb @@ -3,9 +3,14 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + feature_category :service_desk sidekiq_options retry: 3 + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + tags :needs_own_queue + def should_perform? ::Gitlab::ServiceDeskEmail.enabled? end diff --git a/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb b/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb index 88b060a454a..15669ac5852 100644 --- a/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb +++ b/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb @@ -4,6 +4,8 @@ module Snippets class ScheduleBulkRepositoryShardMovesWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 idempotent! diff --git a/app/workers/ssh_keys/expired_notification_worker.rb b/app/workers/ssh_keys/expired_notification_worker.rb index b67849942b0..6afeecdd1b5 100644 --- a/app/workers/ssh_keys/expired_notification_worker.rb +++ b/app/workers/ssh_keys/expired_notification_worker.rb @@ -4,7 +4,8 @@ module SshKeys class ExpiredNotificationWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :compliance_management diff --git a/app/workers/ssh_keys/expiring_soon_notification_worker.rb b/app/workers/ssh_keys/expiring_soon_notification_worker.rb index d87e31c36a5..ef256621e07 100644 --- a/app/workers/ssh_keys/expiring_soon_notification_worker.rb +++ b/app/workers/ssh_keys/expiring_soon_notification_worker.rb @@ -4,7 +4,8 @@ module SshKeys class ExpiringSoonNotificationWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :compliance_management diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb index e206a51a417..e0d8958fc80 100644 --- a/app/workers/stage_update_worker.rb +++ b/app/workers/stage_update_worker.rb @@ -3,6 +3,8 @@ class StageUpdateWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index b3b3d6e7554..5723380a3f3 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -3,7 +3,8 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :continuous_integration diff --git a/app/workers/stuck_export_jobs_worker.rb b/app/workers/stuck_export_jobs_worker.rb index 398f2c915a9..486d40c443a 100644 --- a/app/workers/stuck_export_jobs_worker.rb +++ b/app/workers/stuck_export_jobs_worker.rb @@ -4,7 +4,8 @@ class StuckExportJobsWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker updates export states inline and does not schedule # other jobs. diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb index e50b218e1f6..3a650fcb7bf 100644 --- a/app/workers/stuck_merge_jobs_worker.rb +++ b/app/workers/stuck_merge_jobs_worker.rb @@ -3,7 +3,8 @@ class StuckMergeJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :code_review diff --git a/app/workers/system_hook_push_worker.rb b/app/workers/system_hook_push_worker.rb index 8c801f2bed8..02239c6e764 100644 --- a/app/workers/system_hook_push_worker.rb +++ b/app/workers/system_hook_push_worker.rb @@ -3,6 +3,8 @@ class SystemHookPushWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/todos_destroyer/confidential_issue_worker.rb b/app/workers/todos_destroyer/confidential_issue_worker.rb index 8a43ea3c2e0..91a8777fe9b 100644 --- a/app/workers/todos_destroyer/confidential_issue_worker.rb +++ b/app/workers/todos_destroyer/confidential_issue_worker.rb @@ -4,6 +4,8 @@ module TodosDestroyer class ConfidentialIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/destroyed_issuable_worker.rb b/app/workers/todos_destroyer/destroyed_issuable_worker.rb index a3a8147095e..ff4f5e15472 100644 --- a/app/workers/todos_destroyer/destroyed_issuable_worker.rb +++ b/app/workers/todos_destroyer/destroyed_issuable_worker.rb @@ -4,6 +4,8 @@ module TodosDestroyer class DestroyedIssuableWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/entity_leave_worker.rb b/app/workers/todos_destroyer/entity_leave_worker.rb index 166d8701f7a..8e3944e0a23 100644 --- a/app/workers/todos_destroyer/entity_leave_worker.rb +++ b/app/workers/todos_destroyer/entity_leave_worker.rb @@ -4,6 +4,8 @@ module TodosDestroyer class EntityLeaveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/group_private_worker.rb b/app/workers/todos_destroyer/group_private_worker.rb index 30d1f74fb28..76bf034037d 100644 --- a/app/workers/todos_destroyer/group_private_worker.rb +++ b/app/workers/todos_destroyer/group_private_worker.rb @@ -4,6 +4,8 @@ module TodosDestroyer class GroupPrivateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/private_features_worker.rb b/app/workers/todos_destroyer/private_features_worker.rb index d6a4260a464..150e1c8a50e 100644 --- a/app/workers/todos_destroyer/private_features_worker.rb +++ b/app/workers/todos_destroyer/private_features_worker.rb @@ -4,6 +4,8 @@ module TodosDestroyer class PrivateFeaturesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/project_private_worker.rb b/app/workers/todos_destroyer/project_private_worker.rb index c4fed03f11a..3d86df0c548 100644 --- a/app/workers/todos_destroyer/project_private_worker.rb +++ b/app/workers/todos_destroyer/project_private_worker.rb @@ -4,6 +4,8 @@ module TodosDestroyer class ProjectPrivateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include TodosDestroyerQueue diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb index 8322110b753..73af4444d60 100644 --- a/app/workers/trending_projects_worker.rb +++ b/app/workers/trending_projects_worker.rb @@ -3,7 +3,8 @@ class TrendingProjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :source_code_management diff --git a/app/workers/update_container_registry_info_worker.rb b/app/workers/update_container_registry_info_worker.rb index cf08c650d0d..d94b7ac1a72 100644 --- a/app/workers/update_container_registry_info_worker.rb +++ b/app/workers/update_container_registry_info_worker.rb @@ -3,7 +3,8 @@ class UpdateContainerRegistryInfoWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :container_registry diff --git a/app/workers/update_external_pull_requests_worker.rb b/app/workers/update_external_pull_requests_worker.rb index ee47cbd6523..2a319a7f6f9 100644 --- a/app/workers/update_external_pull_requests_worker.rb +++ b/app/workers/update_external_pull_requests_worker.rb @@ -3,6 +3,8 @@ class UpdateExternalPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/update_head_pipeline_for_merge_request_worker.rb b/app/workers/update_head_pipeline_for_merge_request_worker.rb index f1dd250f432..61fe278e016 100644 --- a/app/workers/update_head_pipeline_for_merge_request_worker.rb +++ b/app/workers/update_head_pipeline_for_merge_request_worker.rb @@ -3,6 +3,8 @@ class UpdateHeadPipelineForMergeRequestWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include PipelineQueue diff --git a/app/workers/update_highest_role_worker.rb b/app/workers/update_highest_role_worker.rb index cecf3f99b50..d5df46c172b 100644 --- a/app/workers/update_highest_role_worker.rb +++ b/app/workers/update_highest_role_worker.rb @@ -3,6 +3,8 @@ class UpdateHighestRoleWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :utilization diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index 6f86a7e7e2f..421a6e47425 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -3,6 +3,8 @@ class UpdateMergeRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :code_review diff --git a/app/workers/update_project_statistics_worker.rb b/app/workers/update_project_statistics_worker.rb index c93c32f4e75..4a32753ac70 100644 --- a/app/workers/update_project_statistics_worker.rb +++ b/app/workers/update_project_statistics_worker.rb @@ -4,6 +4,8 @@ class UpdateProjectStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/upload_checksum_worker.rb b/app/workers/upload_checksum_worker.rb index 765e3a63e75..8e250f38e2c 100644 --- a/app/workers/upload_checksum_worker.rb +++ b/app/workers/upload_checksum_worker.rb @@ -3,6 +3,8 @@ class UploadChecksumWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :geo_replication diff --git a/app/workers/user_status_cleanup/batch_worker.rb b/app/workers/user_status_cleanup/batch_worker.rb index f46b4119f9b..b6ca6548572 100644 --- a/app/workers/user_status_cleanup/batch_worker.rb +++ b/app/workers/user_status_cleanup/batch_worker.rb @@ -5,7 +5,8 @@ module UserStatusCleanup class BatchWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext include CronjobQueue # rubocop:enable Scalability/CronWorkerContext diff --git a/app/workers/users/create_statistics_worker.rb b/app/workers/users/create_statistics_worker.rb index e44039f2016..0c27d165ded 100644 --- a/app/workers/users/create_statistics_worker.rb +++ b/app/workers/users/create_statistics_worker.rb @@ -4,7 +4,8 @@ module Users class CreateStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/users/deactivate_dormant_users_worker.rb b/app/workers/users/deactivate_dormant_users_worker.rb index e583823312f..bcb13483379 100644 --- a/app/workers/users/deactivate_dormant_users_worker.rb +++ b/app/workers/users/deactivate_dormant_users_worker.rb @@ -4,6 +4,8 @@ module Users class DeactivateDormantUsersWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + include CronjobQueue feature_category :utilization diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb index 525a72e02ef..af351c3c207 100644 --- a/app/workers/wait_for_cluster_creation_worker.rb +++ b/app/workers/wait_for_cluster_creation_worker.rb @@ -3,6 +3,8 @@ class WaitForClusterCreationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ClusterQueue diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index a2a53ca922a..5b4567dde29 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -6,16 +6,18 @@ class WebHookWorker include ApplicationWorker feature_category :integrations - worker_has_external_dependencies! loggable_arguments 2 data_consistency :delayed - sidekiq_options retry: 4, dead: false + urgency :low + + worker_has_external_dependencies! def perform(hook_id, data, hook_name) - hook = WebHook.find(hook_id) - data = data.with_indifferent_access + hook = WebHook.find_by_id(hook_id) + return unless hook + data = data.with_indifferent_access WebHookService.new(hook, data, hook_name, jid).execute end end diff --git a/app/workers/web_hooks/destroy_worker.rb b/app/workers/web_hooks/destroy_worker.rb index c1886576c41..b92fe86bafb 100644 --- a/app/workers/web_hooks/destroy_worker.rb +++ b/app/workers/web_hooks/destroy_worker.rb @@ -4,11 +4,12 @@ module WebHooks class DestroyWorker include ApplicationWorker + data_consistency :always sidekiq_options retry: 3 - feature_category :integrations tags :exclude_from_kubernetes urgency :low + idempotent! def perform(user_id, web_hook_id) diff --git a/app/workers/web_hooks/log_execution_worker.rb b/app/workers/web_hooks/log_execution_worker.rb index 58059370200..50d91182c80 100644 --- a/app/workers/web_hooks/log_execution_worker.rb +++ b/app/workers/web_hooks/log_execution_worker.rb @@ -4,10 +4,12 @@ module WebHooks class LogExecutionWorker include ApplicationWorker - idempotent! + data_consistency :always feature_category :integrations urgency :low + idempotent! + # This worker accepts an extra argument. This enables us to # treat this worker as idempotent. Currently this is set to # the Job ID (jid) of the parent worker. diff --git a/app/workers/x509_certificate_revoke_worker.rb b/app/workers/x509_certificate_revoke_worker.rb index cbf9fbb7525..3e170c9df22 100644 --- a/app/workers/x509_certificate_revoke_worker.rb +++ b/app/workers/x509_certificate_revoke_worker.rb @@ -3,6 +3,8 @@ class X509CertificateRevokeWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 feature_category :source_code_management diff --git a/app/workers/x509_issuer_crl_check_worker.rb b/app/workers/x509_issuer_crl_check_worker.rb index d66ad6c1c15..39440504927 100644 --- a/app/workers/x509_issuer_crl_check_worker.rb +++ b/app/workers/x509_issuer_crl_check_worker.rb @@ -3,7 +3,8 @@ class X509IssuerCrlCheckWorker include ApplicationWorker - sidekiq_options retry: 3 + data_consistency :always + include CronjobQueue feature_category :source_code_management |