diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
| commit | 4555e1b21c365ed8303ffb7a3325d773c9b8bf31 (patch) | |
| tree | 5423a1c7516cffe36384133ade12572cf709398d /app/workers | |
| parent | e570267f2f6b326480d284e0164a6464ba4081bc (diff) | |
| download | gitlab-ce-4555e1b21c365ed8303ffb7a3325d773c9b8bf31.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42
Diffstat (limited to 'app/workers')
295 files changed, 1882 insertions, 357 deletions
diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb index 8d589c03259..ea7709c649f 100644 --- a/app/workers/admin_email_worker.rb +++ b/app/workers/admin_email_worker.rb @@ -2,6 +2,8 @@ class AdminEmailWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index fa6ea54e342..07c1ce0d939 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -4,6 +4,7 @@ # Do not edit it manually! --- - :name: authorized_project_update:authorized_project_update_project_create + :worker_name: AuthorizedProjectUpdate::ProjectCreateWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -12,6 +13,7 @@ :idempotent: true :tags: [] - :name: authorized_project_update:authorized_project_update_project_group_link_create + :worker_name: AuthorizedProjectUpdate::ProjectGroupLinkCreateWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -20,6 +22,7 @@ :idempotent: true :tags: [] - :name: authorized_project_update:authorized_project_update_user_refresh_over_user_range + :worker_name: AuthorizedProjectUpdate::UserRefreshOverUserRangeWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -28,6 +31,7 @@ :idempotent: :tags: [] - :name: authorized_project_update:authorized_project_update_user_refresh_with_low_urgency + :worker_name: AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -36,6 +40,7 @@ :idempotent: true :tags: [] - :name: auto_devops:auto_devops_disable + :worker_name: AutoDevops::DisableWorker :feature_category: :auto_devops :has_external_dependencies: :urgency: :low @@ -44,6 +49,7 @@ :idempotent: :tags: [] - :name: auto_merge:auto_merge_process + :worker_name: AutoMergeProcessWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -52,46 +58,57 @@ :idempotent: :tags: [] - :name: chaos:chaos_cpu_spin + :worker_name: Chaos::CpuSpinWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: chaos:chaos_db_spin + :worker_name: Chaos::DbSpinWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: chaos:chaos_kill + :worker_name: Chaos::KillWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: chaos:chaos_leak_mem + :worker_name: Chaos::LeakMemWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: chaos:chaos_sleep + :worker_name: Chaos::SleepWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: container_repository:cleanup_container_repository + :worker_name: CleanupContainerRepositoryWorker :feature_category: :container_registry :has_external_dependencies: :urgency: :low @@ -100,14 +117,17 @@ :idempotent: true :tags: [] - :name: container_repository:container_expiration_policies_cleanup_container_repository + :worker_name: ContainerExpirationPolicies::CleanupContainerRepositoryWorker :feature_category: :container_registry :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: container_repository:delete_container_repository + :worker_name: DeleteContainerRepositoryWorker :feature_category: :container_registry :has_external_dependencies: :urgency: :low @@ -116,6 +136,7 @@ :idempotent: :tags: [] - :name: cronjob:admin_email + :worker_name: AdminEmailWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -124,22 +145,27 @@ :idempotent: :tags: [] - :name: cronjob:analytics_instance_statistics_count_job_trigger + :worker_name: Analytics::InstanceStatistics::CountJobTriggerWorker :feature_category: :devops_reports :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:analytics_usage_trends_count_job_trigger + :worker_name: Analytics::UsageTrends::CountJobTriggerWorker :feature_category: :devops_reports :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:authorized_project_update_periodic_recalculate + :worker_name: AuthorizedProjectUpdate::PeriodicRecalculateWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -148,6 +174,7 @@ :idempotent: true :tags: [] - :name: cronjob:ci_archive_traces_cron + :worker_name: Ci::ArchiveTracesCronWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -155,7 +182,8 @@ :weight: 1 :idempotent: :tags: [] -- :name: cronjob:ci_pipeline_artifacts_expire_artifacts +- :name: cronjob:ci_delete_unit_tests + :worker_name: Ci::DeleteUnitTestsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -163,7 +191,18 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:ci_pipeline_artifacts_expire_artifacts + :worker_name: Ci::PipelineArtifacts::ExpireArtifactsWorker + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: + - :exclude_from_kubernetes - :name: cronjob:ci_platform_metrics_update_cron + :worker_name: CiPlatformMetricsUpdateCronWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -172,14 +211,17 @@ :idempotent: :tags: [] - :name: cronjob:ci_schedule_delete_objects_cron + :worker_name: Ci::ScheduleDeleteObjectsCronWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:container_expiration_policy + :worker_name: ContainerExpirationPolicyWorker :feature_category: :container_registry :has_external_dependencies: :urgency: :low @@ -188,14 +230,17 @@ :idempotent: :tags: [] - :name: cronjob:database_batched_background_migration + :worker_name: Database::BatchedBackgroundMigrationWorker :feature_category: :database :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:environments_auto_stop_cron + :worker_name: Environments::AutoStopCronWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -204,6 +249,7 @@ :idempotent: :tags: [] - :name: cronjob:expire_build_artifacts + :worker_name: ExpireBuildArtifactsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -212,6 +258,7 @@ :idempotent: :tags: [] - :name: cronjob:gitlab_usage_ping + :worker_name: GitlabUsagePingWorker :feature_category: :usage_ping :has_external_dependencies: :urgency: :low @@ -220,6 +267,7 @@ :idempotent: :tags: [] - :name: cronjob:import_export_project_cleanup + :worker_name: ImportExportProjectCleanupWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -228,6 +276,7 @@ :idempotent: :tags: [] - :name: cronjob:import_stuck_project_import_jobs + :worker_name: Gitlab::Import::StuckProjectImportJobsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -236,6 +285,7 @@ :idempotent: :tags: [] - :name: cronjob:issue_due_scheduler + :worker_name: IssueDueSchedulerWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -244,6 +294,7 @@ :idempotent: :tags: [] - :name: cronjob:jira_import_stuck_jira_import_jobs + :worker_name: Gitlab::JiraImport::StuckJiraImportJobsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -252,14 +303,17 @@ :idempotent: :tags: [] - :name: cronjob:member_invitation_reminder_emails + :worker_name: MemberInvitationReminderEmailsWorker :feature_category: :subgroups :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:metrics_dashboard_schedule_annotations_prune + :worker_name: Metrics::Dashboard::ScheduleAnnotationsPruneWorker :feature_category: :metrics :has_external_dependencies: :urgency: :low @@ -268,14 +322,17 @@ :idempotent: true :tags: [] - :name: cronjob:namespaces_in_product_marketing_emails + :worker_name: Namespaces::InProductMarketingEmailsWorker :feature_category: :subgroups :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:namespaces_prune_aggregation_schedules + :worker_name: Namespaces::PruneAggregationSchedulesWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -284,14 +341,17 @@ :idempotent: :tags: [] - :name: cronjob:packages_composer_cache_cleanup + :worker_name: Packages::Composer::CacheCleanupWorker :feature_category: :package_registry :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:pages_domain_removal_cron + :worker_name: PagesDomainRemovalCronWorker :feature_category: :pages :has_external_dependencies: :urgency: :low @@ -300,6 +360,7 @@ :idempotent: :tags: [] - :name: cronjob:pages_domain_ssl_renewal_cron + :worker_name: PagesDomainSslRenewalCronWorker :feature_category: :pages :has_external_dependencies: :urgency: :low @@ -308,6 +369,7 @@ :idempotent: :tags: [] - :name: cronjob:pages_domain_verification_cron + :worker_name: PagesDomainVerificationCronWorker :feature_category: :pages :has_external_dependencies: :urgency: :low @@ -316,6 +378,7 @@ :idempotent: :tags: [] - :name: cronjob:partition_creation + :worker_name: PartitionCreationWorker :feature_category: :database :has_external_dependencies: :urgency: :low @@ -324,14 +387,17 @@ :idempotent: true :tags: [] - :name: cronjob:personal_access_tokens_expired_notification + :worker_name: PersonalAccessTokens::ExpiredNotificationWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:personal_access_tokens_expiring + :worker_name: PersonalAccessTokens::ExpiringWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -340,6 +406,7 @@ :idempotent: :tags: [] - :name: cronjob:pipeline_schedule + :worker_name: PipelineScheduleWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -348,6 +415,7 @@ :idempotent: :tags: [] - :name: cronjob:prune_old_events + :worker_name: PruneOldEventsWorker :feature_category: :users :has_external_dependencies: :urgency: :low @@ -356,6 +424,7 @@ :idempotent: :tags: [] - :name: cronjob:prune_web_hook_logs + :worker_name: PruneWebHookLogsWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low @@ -364,14 +433,17 @@ :idempotent: :tags: [] - :name: cronjob:releases_manage_evidence + :worker_name: Releases::ManageEvidenceWorker :feature_category: :release_evidence :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:remove_expired_group_links + :worker_name: RemoveExpiredGroupLinksWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -380,6 +452,7 @@ :idempotent: :tags: [] - :name: cronjob:remove_expired_members + :worker_name: RemoveExpiredMembersWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -388,14 +461,17 @@ :idempotent: :tags: [] - :name: cronjob:remove_unaccepted_member_invites + :worker_name: RemoveUnacceptedMemberInvitesWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:remove_unreferenced_lfs_objects + :worker_name: RemoveUnreferencedLfsObjectsWorker :feature_category: :git_lfs :has_external_dependencies: :urgency: :low @@ -404,6 +480,7 @@ :idempotent: :tags: [] - :name: cronjob:repository_archive_cache + :worker_name: RepositoryArchiveCacheWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -412,6 +489,7 @@ :idempotent: :tags: [] - :name: cronjob:repository_check_dispatch + :worker_name: RepositoryCheck::DispatchWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -420,6 +498,7 @@ :idempotent: :tags: [] - :name: cronjob:requests_profiles + :worker_name: RequestsProfilesWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -428,14 +507,17 @@ :idempotent: :tags: [] - :name: cronjob:schedule_merge_request_cleanup_refs + :worker_name: ScheduleMergeRequestCleanupRefsWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:schedule_migrate_external_diffs + :worker_name: ScheduleMigrateExternalDiffsWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low @@ -444,22 +526,27 @@ :idempotent: :tags: [] - :name: cronjob:ssh_keys_expired_notification + :worker_name: SshKeys::ExpiredNotificationWorker :feature_category: :compliance_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:ssh_keys_expiring_soon_notification + :worker_name: SshKeys::ExpiringSoonNotificationWorker :feature_category: :compliance_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:stuck_ci_jobs + :worker_name: StuckCiJobsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -468,6 +555,7 @@ :idempotent: :tags: [] - :name: cronjob:stuck_export_jobs + :worker_name: StuckExportJobsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -476,6 +564,7 @@ :idempotent: :tags: [] - :name: cronjob:stuck_merge_jobs + :worker_name: StuckMergeJobsWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low @@ -484,6 +573,7 @@ :idempotent: :tags: [] - :name: cronjob:trending_projects + :worker_name: TrendingProjectsWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -492,6 +582,7 @@ :idempotent: :tags: [] - :name: cronjob:update_container_registry_info + :worker_name: UpdateContainerRegistryInfoWorker :feature_category: :container_registry :has_external_dependencies: :urgency: :low @@ -500,14 +591,17 @@ :idempotent: true :tags: [] - :name: cronjob:user_status_cleanup_batch + :worker_name: UserStatusCleanup::BatchWorker :feature_category: :users :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: cronjob:users_create_statistics + :worker_name: Users::CreateStatisticsWorker :feature_category: :users :has_external_dependencies: :urgency: :low @@ -515,7 +609,18 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:users_deactivate_dormant_users + :worker_name: Users::DeactivateDormantUsersWorker + :feature_category: :utilization + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: + - :exclude_from_kubernetes - :name: cronjob:x509_issuer_crl_check + :worker_name: X509IssuerCrlCheckWorker :feature_category: :source_code_management :has_external_dependencies: true :urgency: :low @@ -524,6 +629,7 @@ :idempotent: true :tags: [] - :name: dependency_proxy:purge_dependency_proxy_cache + :worker_name: PurgeDependencyProxyCacheWorker :feature_category: :dependency_proxy :has_external_dependencies: :urgency: :low @@ -532,14 +638,17 @@ :idempotent: true :tags: [] - :name: deployment:deployments_drop_older_deployments + :worker_name: Deployments::DropOlderDeploymentsWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 3 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: deployment:deployments_execute_hooks + :worker_name: Deployments::ExecuteHooksWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -548,6 +657,7 @@ :idempotent: :tags: [] - :name: deployment:deployments_finished + :worker_name: Deployments::FinishedWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -556,6 +666,16 @@ :idempotent: :tags: [] - :name: deployment:deployments_forward_deployment + :worker_name: Deployments::ForwardDeploymentWorker + :feature_category: :continuous_delivery + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 3 + :idempotent: + :tags: [] +- :name: deployment:deployments_hooks + :worker_name: Deployments::HooksWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -564,6 +684,7 @@ :idempotent: :tags: [] - :name: deployment:deployments_link_merge_request + :worker_name: Deployments::LinkMergeRequestWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -572,6 +693,7 @@ :idempotent: true :tags: [] - :name: deployment:deployments_success + :worker_name: Deployments::SuccessWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -580,6 +702,7 @@ :idempotent: :tags: [] - :name: deployment:deployments_update_environment + :worker_name: Deployments::UpdateEnvironmentWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -588,6 +711,7 @@ :idempotent: true :tags: [] - :name: gcp_cluster:cluster_configure_istio + :worker_name: ClusterConfigureIstioWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -596,6 +720,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_install_app + :worker_name: ClusterInstallAppWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -604,6 +729,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_patch_app + :worker_name: ClusterPatchAppWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -612,6 +738,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_provision + :worker_name: ClusterProvisionWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -620,6 +747,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_update_app + :worker_name: ClusterUpdateAppWorker :feature_category: :kubernetes_management :has_external_dependencies: :urgency: :low @@ -628,6 +756,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_upgrade_app + :worker_name: ClusterUpgradeAppWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -636,6 +765,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_wait_for_app_installation + :worker_name: ClusterWaitForAppInstallationWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -644,6 +774,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_wait_for_app_update + :worker_name: ClusterWaitForAppUpdateWorker :feature_category: :kubernetes_management :has_external_dependencies: :urgency: :low @@ -652,6 +783,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:cluster_wait_for_ingress_ip_address + :worker_name: ClusterWaitForIngressIpAddressWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -660,6 +792,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:clusters_applications_activate_service + :worker_name: Clusters::Applications::ActivateServiceWorker :feature_category: :kubernetes_management :has_external_dependencies: :urgency: :low @@ -668,6 +801,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:clusters_applications_deactivate_service + :worker_name: Clusters::Applications::DeactivateServiceWorker :feature_category: :kubernetes_management :has_external_dependencies: :urgency: :low @@ -676,6 +810,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:clusters_applications_uninstall + :worker_name: Clusters::Applications::UninstallWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -684,6 +819,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:clusters_applications_wait_for_uninstall_app + :worker_name: Clusters::Applications::WaitForUninstallAppWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -692,6 +828,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:clusters_cleanup_app + :worker_name: Clusters::Cleanup::AppWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -700,6 +837,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:clusters_cleanup_project_namespace + :worker_name: Clusters::Cleanup::ProjectNamespaceWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -708,6 +846,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:clusters_cleanup_service_account + :worker_name: Clusters::Cleanup::ServiceAccountWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -716,6 +855,7 @@ :idempotent: :tags: [] - :name: gcp_cluster:wait_for_cluster_creation + :worker_name: WaitForClusterCreationWorker :feature_category: :kubernetes_management :has_external_dependencies: true :urgency: :low @@ -724,6 +864,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_import_diff_note + :worker_name: Gitlab::GithubImport::ImportDiffNoteWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low @@ -732,6 +873,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_import_issue + :worker_name: Gitlab::GithubImport::ImportIssueWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low @@ -740,6 +882,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_import_lfs_object + :worker_name: Gitlab::GithubImport::ImportLfsObjectWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low @@ -748,6 +891,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_import_note + :worker_name: Gitlab::GithubImport::ImportNoteWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low @@ -756,6 +900,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_import_pull_request + :worker_name: Gitlab::GithubImport::ImportPullRequestWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low @@ -764,22 +909,27 @@ :idempotent: :tags: [] - :name: github_importer:github_import_import_pull_request_merged_by + :worker_name: Gitlab::GithubImport::ImportPullRequestMergedByWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: github_importer:github_import_import_pull_request_review + :worker_name: Gitlab::GithubImport::ImportPullRequestReviewWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: github_importer:github_import_refresh_import_jid + :worker_name: Gitlab::GithubImport::RefreshImportJidWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -788,6 +938,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_stage_finish_import + :worker_name: Gitlab::GithubImport::Stage::FinishImportWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -796,6 +947,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_stage_import_base_data + :worker_name: Gitlab::GithubImport::Stage::ImportBaseDataWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -804,6 +956,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_stage_import_issues_and_diff_notes + :worker_name: Gitlab::GithubImport::Stage::ImportIssuesAndDiffNotesWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -812,6 +965,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_stage_import_lfs_objects + :worker_name: Gitlab::GithubImport::Stage::ImportLfsObjectsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -820,6 +974,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_stage_import_notes + :worker_name: Gitlab::GithubImport::Stage::ImportNotesWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -828,6 +983,7 @@ :idempotent: :tags: [] - :name: github_importer:github_import_stage_import_pull_requests + :worker_name: Gitlab::GithubImport::Stage::ImportPullRequestsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -836,22 +992,27 @@ :idempotent: :tags: [] - :name: github_importer:github_import_stage_import_pull_requests_merged_by + :worker_name: Gitlab::GithubImport::Stage::ImportPullRequestsMergedByWorker :feature_category: :importers :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: github_importer:github_import_stage_import_pull_requests_reviews + :worker_name: Gitlab::GithubImport::Stage::ImportPullRequestsReviewsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: github_importer:github_import_stage_import_repository + :worker_name: Gitlab::GithubImport::Stage::ImportRepositoryWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -860,38 +1021,47 @@ :idempotent: :tags: [] - :name: hashed_storage:hashed_storage_migrator + :worker_name: HashedStorage::MigratorWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: hashed_storage:hashed_storage_project_migrate + :worker_name: HashedStorage::ProjectMigrateWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: hashed_storage:hashed_storage_project_rollback + :worker_name: HashedStorage::ProjectRollbackWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: hashed_storage:hashed_storage_rollbacker + :worker_name: HashedStorage::RollbackerWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_gitlab_com - :name: incident_management:clusters_applications_check_prometheus_health + :worker_name: Clusters::Applications::CheckPrometheusHealthWorker :feature_category: :incident_management :has_external_dependencies: true :urgency: :low @@ -900,14 +1070,17 @@ :idempotent: true :tags: [] - :name: incident_management:incident_management_add_severity_system_note + :worker_name: IncidentManagement::AddSeveritySystemNoteWorker :feature_category: :incident_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: incident_management:incident_management_pager_duty_process_incident + :worker_name: IncidentManagement::PagerDuty::ProcessIncidentWorker :feature_category: :incident_management :has_external_dependencies: :urgency: :low @@ -916,6 +1089,7 @@ :idempotent: :tags: [] - :name: incident_management:incident_management_process_alert + :worker_name: IncidentManagement::ProcessAlertWorker :feature_category: :incident_management :has_external_dependencies: :urgency: :low @@ -923,7 +1097,17 @@ :weight: 2 :idempotent: :tags: [] +- :name: incident_management:incident_management_process_alert_worker_v2 + :worker_name: IncidentManagement::ProcessAlertWorkerV2 + :feature_category: :incident_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 2 + :idempotent: true + :tags: [] - :name: incident_management:incident_management_process_prometheus_alert + :worker_name: IncidentManagement::ProcessPrometheusAlertWorker :feature_category: :incident_management :has_external_dependencies: :urgency: :low @@ -932,6 +1116,7 @@ :idempotent: :tags: [] - :name: jira_connect:jira_connect_sync_branch + :worker_name: JiraConnect::SyncBranchWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low @@ -940,30 +1125,37 @@ :idempotent: true :tags: [] - :name: jira_connect:jira_connect_sync_builds + :worker_name: JiraConnect::SyncBuildsWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: jira_connect:jira_connect_sync_deployments + :worker_name: JiraConnect::SyncDeploymentsWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: jira_connect:jira_connect_sync_feature_flags + :worker_name: JiraConnect::SyncFeatureFlagsWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: jira_connect:jira_connect_sync_merge_request + :worker_name: JiraConnect::SyncMergeRequestWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low @@ -972,14 +1164,17 @@ :idempotent: true :tags: [] - :name: jira_connect:jira_connect_sync_project + :worker_name: JiraConnect::SyncProjectWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: jira_importer:jira_import_advance_stage + :worker_name: Gitlab::JiraImport::AdvanceStageWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -988,6 +1183,7 @@ :idempotent: :tags: [] - :name: jira_importer:jira_import_import_issue + :worker_name: Gitlab::JiraImport::ImportIssueWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -996,6 +1192,7 @@ :idempotent: :tags: [] - :name: jira_importer:jira_import_stage_finish_import + :worker_name: Gitlab::JiraImport::Stage::FinishImportWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1004,6 +1201,7 @@ :idempotent: :tags: [] - :name: jira_importer:jira_import_stage_import_attachments + :worker_name: Gitlab::JiraImport::Stage::ImportAttachmentsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1012,6 +1210,7 @@ :idempotent: :tags: [] - :name: jira_importer:jira_import_stage_import_issues + :worker_name: Gitlab::JiraImport::Stage::ImportIssuesWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1020,6 +1219,7 @@ :idempotent: :tags: [] - :name: jira_importer:jira_import_stage_import_labels + :worker_name: Gitlab::JiraImport::Stage::ImportLabelsWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1028,6 +1228,7 @@ :idempotent: :tags: [] - :name: jira_importer:jira_import_stage_import_notes + :worker_name: Gitlab::JiraImport::Stage::ImportNotesWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1036,6 +1237,7 @@ :idempotent: :tags: [] - :name: jira_importer:jira_import_stage_start_import + :worker_name: Gitlab::JiraImport::Stage::StartImportWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1044,6 +1246,7 @@ :idempotent: :tags: [] - :name: mail_scheduler:mail_scheduler_issue_due + :worker_name: MailScheduler::IssueDueWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1052,6 +1255,7 @@ :idempotent: :tags: [] - :name: mail_scheduler:mail_scheduler_notification_service + :worker_name: MailScheduler::NotificationServiceWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1060,6 +1264,7 @@ :idempotent: :tags: [] - :name: object_pool:object_pool_create + :worker_name: ObjectPool::CreateWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :low @@ -1068,6 +1273,7 @@ :idempotent: :tags: [] - :name: object_pool:object_pool_destroy + :worker_name: ObjectPool::DestroyWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :low @@ -1076,6 +1282,7 @@ :idempotent: :tags: [] - :name: object_pool:object_pool_join + :worker_name: ObjectPool::JoinWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :low @@ -1084,6 +1291,7 @@ :idempotent: :tags: [] - :name: object_pool:object_pool_schedule_join + :worker_name: ObjectPool::ScheduleJoinWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :low @@ -1092,6 +1300,7 @@ :idempotent: :tags: [] - :name: object_storage:object_storage_background_move + :worker_name: ObjectStorage::BackgroundMoveWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low @@ -1100,6 +1309,7 @@ :idempotent: :tags: [] - :name: object_storage:object_storage_migrate_uploads + :worker_name: ObjectStorage::MigrateUploadsWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low @@ -1107,23 +1317,38 @@ :weight: 1 :idempotent: :tags: [] +- :name: package_repositories:packages_debian_process_changes + :worker_name: Packages::Debian::ProcessChangesWorker + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: + - :exclude_from_kubernetes - :name: package_repositories:packages_go_sync_packages + :worker_name: Packages::Go::SyncPackagesWorker :feature_category: :package_registry :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: package_repositories:packages_maven_metadata_sync + :worker_name: Packages::Maven::Metadata::SyncWorker :feature_category: :package_registry :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: package_repositories:packages_nuget_extraction + :worker_name: Packages::Nuget::ExtractionWorker :feature_category: :package_registry :has_external_dependencies: :urgency: :low @@ -1132,14 +1357,17 @@ :idempotent: :tags: [] - :name: package_repositories:packages_rubygems_extraction + :worker_name: Packages::Rubygems::ExtractionWorker :feature_category: :package_registry :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: true - :tags: [] + :idempotent: + :tags: + - :exclude_from_kubernetes - :name: pipeline_background:archive_trace + :worker_name: ArchiveTraceWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1148,6 +1376,7 @@ :idempotent: :tags: [] - :name: pipeline_background:ci_build_trace_chunk_flush + :worker_name: Ci::BuildTraceChunkFlushWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1156,6 +1385,7 @@ :idempotent: true :tags: [] - :name: pipeline_background:ci_daily_build_group_report_results + :worker_name: Ci::DailyBuildGroupReportResultsWorker :feature_category: :code_testing :has_external_dependencies: :urgency: :low @@ -1164,22 +1394,27 @@ :idempotent: true :tags: [] - :name: pipeline_background:ci_pipeline_artifacts_coverage_report + :worker_name: Ci::PipelineArtifacts::CoverageReportWorker :feature_category: :code_testing :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pipeline_background:ci_pipeline_artifacts_create_quality_report + :worker_name: Ci::PipelineArtifacts::CreateQualityReportWorker :feature_category: :code_testing :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pipeline_background:ci_pipeline_success_unlock_artifacts + :worker_name: Ci::PipelineSuccessUnlockArtifactsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1188,6 +1423,7 @@ :idempotent: true :tags: [] - :name: pipeline_background:ci_ref_delete_unlock_artifacts + :worker_name: Ci::RefDeleteUnlockArtifactsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1196,14 +1432,17 @@ :idempotent: true :tags: [] - :name: pipeline_background:ci_test_failure_history + :worker_name: Ci::TestFailureHistoryWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pipeline_cache:expire_job_cache + :worker_name: ExpireJobCacheWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1212,6 +1451,7 @@ :idempotent: true :tags: [] - :name: pipeline_cache:expire_pipeline_cache + :worker_name: ExpirePipelineCacheWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1220,6 +1460,7 @@ :idempotent: true :tags: [] - :name: pipeline_creation:create_pipeline + :worker_name: CreatePipelineWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1228,6 +1469,7 @@ :idempotent: :tags: [] - :name: pipeline_creation:merge_requests_create_pipeline + :worker_name: MergeRequests::CreatePipelineWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1236,6 +1478,7 @@ :idempotent: true :tags: [] - :name: pipeline_creation:run_pipeline_schedule + :worker_name: RunPipelineScheduleWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1244,6 +1487,7 @@ :idempotent: :tags: [] - :name: pipeline_default:ci_create_cross_project_pipeline + :worker_name: Ci::CreateCrossProjectPipelineWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1252,22 +1496,36 @@ :idempotent: :tags: [] - :name: pipeline_default:ci_drop_pipeline + :worker_name: Ci::DropPipelineWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 3 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pipeline_default:ci_merge_requests_add_todo_when_build_fails + :worker_name: Ci::MergeRequests::AddTodoWhenBuildFailsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 3 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pipeline_default:ci_pipeline_bridge_status + :worker_name: Ci::PipelineBridgeStatusWorker + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :high + :resource_boundary: :cpu + :weight: 3 + :idempotent: + :tags: [] +- :name: pipeline_default:ci_retry_pipeline + :worker_name: Ci::RetryPipelineWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1276,6 +1534,7 @@ :idempotent: :tags: [] - :name: pipeline_default:pipeline_metrics + :worker_name: PipelineMetricsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1284,6 +1543,7 @@ :idempotent: :tags: [] - :name: pipeline_default:pipeline_notification + :worker_name: PipelineNotificationWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1292,6 +1552,7 @@ :idempotent: :tags: [] - :name: pipeline_hooks:build_hooks + :worker_name: BuildHooksWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1300,6 +1561,7 @@ :idempotent: :tags: [] - :name: pipeline_hooks:pipeline_hooks + :worker_name: PipelineHooksWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1308,6 +1570,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:build_finished + :worker_name: BuildFinishedWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1316,6 +1579,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:build_queue + :worker_name: BuildQueueWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1324,6 +1588,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:build_success + :worker_name: BuildSuccessWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1332,6 +1597,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:ci_build_prepare + :worker_name: Ci::BuildPrepareWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1340,6 +1606,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:ci_build_schedule + :worker_name: Ci::BuildScheduleWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1348,6 +1615,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:ci_initial_pipeline_process + :worker_name: Ci::InitialPipelineProcessWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1356,6 +1624,7 @@ :idempotent: true :tags: [] - :name: pipeline_processing:ci_resource_groups_assign_resource_from_resource_group + :worker_name: Ci::ResourceGroups::AssignResourceFromResourceGroupWorker :feature_category: :continuous_delivery :has_external_dependencies: :urgency: :low @@ -1364,6 +1633,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:pipeline_process + :worker_name: PipelineProcessWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1372,6 +1642,7 @@ :idempotent: :tags: [] - :name: pipeline_processing:pipeline_update + :worker_name: PipelineUpdateWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1380,6 +1651,7 @@ :idempotent: true :tags: [] - :name: pipeline_processing:stage_update + :worker_name: StageUpdateWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1388,6 +1660,7 @@ :idempotent: true :tags: [] - :name: pipeline_processing:update_head_pipeline_for_merge_request + :worker_name: UpdateHeadPipelineForMergeRequestWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :high @@ -1396,6 +1669,7 @@ :idempotent: true :tags: [] - :name: repository_check:repository_check_batch + :worker_name: RepositoryCheck::BatchWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1404,6 +1678,7 @@ :idempotent: :tags: [] - :name: repository_check:repository_check_clear + :worker_name: RepositoryCheck::ClearWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1412,6 +1687,7 @@ :idempotent: :tags: [] - :name: repository_check:repository_check_single_repository + :worker_name: RepositoryCheck::SingleRepositoryWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1420,6 +1696,7 @@ :idempotent: :tags: [] - :name: todos_destroyer:todos_destroyer_confidential_issue + :worker_name: TodosDestroyer::ConfidentialIssueWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1428,14 +1705,17 @@ :idempotent: :tags: [] - :name: todos_destroyer:todos_destroyer_destroyed_issuable + :worker_name: TodosDestroyer::DestroyedIssuableWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: todos_destroyer:todos_destroyer_entity_leave + :worker_name: TodosDestroyer::EntityLeaveWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1444,6 +1724,7 @@ :idempotent: :tags: [] - :name: todos_destroyer:todos_destroyer_group_private + :worker_name: TodosDestroyer::GroupPrivateWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1452,6 +1733,7 @@ :idempotent: :tags: [] - :name: todos_destroyer:todos_destroyer_private_features + :worker_name: TodosDestroyer::PrivateFeaturesWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1460,6 +1742,7 @@ :idempotent: :tags: [] - :name: todos_destroyer:todos_destroyer_project_private + :worker_name: TodosDestroyer::ProjectPrivateWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1468,6 +1751,7 @@ :idempotent: :tags: [] - :name: unassign_issuables:members_destroyer_unassign_issuables + :worker_name: MembersDestroyer::UnassignIssuablesWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -1476,6 +1760,7 @@ :idempotent: true :tags: [] - :name: update_namespace_statistics:namespaces_root_statistics + :worker_name: Namespaces::RootStatisticsWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1484,6 +1769,7 @@ :idempotent: true :tags: [] - :name: update_namespace_statistics:namespaces_schedule_aggregation + :worker_name: Namespaces::ScheduleAggregationWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1492,30 +1778,37 @@ :idempotent: true :tags: [] - :name: analytics_instance_statistics_counter_job + :worker_name: Analytics::InstanceStatistics::CounterJobWorker :feature_category: :devops_reports :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: analytics_usage_trends_counter_job + :worker_name: Analytics::UsageTrends::CounterJobWorker :feature_category: :devops_reports :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: approve_blocked_pending_approval_users + :worker_name: ApproveBlockedPendingApprovalUsersWorker :feature_category: :users :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: authorized_keys + :worker_name: AuthorizedKeysWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :high @@ -1524,6 +1817,7 @@ :idempotent: true :tags: [] - :name: authorized_projects + :worker_name: AuthorizedProjectsWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :high @@ -1532,6 +1826,7 @@ :idempotent: true :tags: [] - :name: background_migration + :worker_name: BackgroundMigrationWorker :feature_category: :database :has_external_dependencies: :urgency: :throttled @@ -1540,30 +1835,56 @@ :idempotent: :tags: [] - :name: bulk_import + :worker_name: BulkImportWorker :feature_category: :importers :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: bulk_imports_entity + :worker_name: BulkImports::EntityWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: + - :exclude_from_kubernetes +- :name: bulk_imports_export_request + :worker_name: BulkImports::ExportRequestWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true :tags: [] - :name: bulk_imports_pipeline + :worker_name: BulkImports::PipelineWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes +- :name: bulk_imports_relation_export + :worker_name: BulkImports::RelationExportWorker + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: + - :exclude_from_kubernetes - :name: chat_notification + :worker_name: ChatNotificationWorker :feature_category: :chatops :has_external_dependencies: true :urgency: :low @@ -1572,14 +1893,17 @@ :idempotent: :tags: [] - :name: ci_delete_objects + :worker_name: Ci::DeleteObjectsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: create_commit_signature + :worker_name: CreateCommitSignatureWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1588,6 +1912,7 @@ :idempotent: true :tags: [] - :name: create_note_diff_file + :worker_name: CreateNoteDiffFileWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low @@ -1596,6 +1921,7 @@ :idempotent: :tags: [] - :name: default + :worker_name: :feature_category: :has_external_dependencies: :urgency: @@ -1604,6 +1930,7 @@ :idempotent: :tags: [] - :name: delete_diff_files + :worker_name: DeleteDiffFilesWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low @@ -1612,6 +1939,7 @@ :idempotent: :tags: [] - :name: delete_merged_branches + :worker_name: DeleteMergedBranchesWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1620,6 +1948,7 @@ :idempotent: :tags: [] - :name: delete_stored_files + :worker_name: DeleteStoredFilesWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low @@ -1628,6 +1957,7 @@ :idempotent: :tags: [] - :name: delete_user + :worker_name: DeleteUserWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -1636,14 +1966,17 @@ :idempotent: :tags: [] - :name: design_management_copy_design_collection + :worker_name: DesignManagement::CopyDesignCollectionWorker :feature_category: :design_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: design_management_new_version + :worker_name: DesignManagement::NewVersionWorker :feature_category: :design_management :has_external_dependencies: :urgency: :low @@ -1652,14 +1985,17 @@ :idempotent: :tags: [] - :name: destroy_pages_deployments + :worker_name: DestroyPagesDeploymentsWorker :feature_category: :pages :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: detect_repository_languages + :worker_name: DetectRepositoryLanguagesWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1668,22 +2004,27 @@ :idempotent: :tags: [] - :name: disallow_two_factor_for_group + :worker_name: DisallowTwoFactorForGroupWorker :feature_category: :subgroups :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: disallow_two_factor_for_subgroups + :worker_name: DisallowTwoFactorForSubgroupsWorker :feature_category: :subgroups :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: email_receiver + :worker_name: EmailReceiverWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :high @@ -1692,6 +2033,7 @@ :idempotent: :tags: [] - :name: emails_on_push + :worker_name: EmailsOnPushWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1700,14 +2042,17 @@ :idempotent: :tags: [] - :name: environments_canary_ingress_update + :worker_name: Environments::CanaryIngress::UpdateWorker :feature_category: :continuous_delivery :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: error_tracking_issue_link + :worker_name: ErrorTrackingIssueLinkWorker :feature_category: :error_tracking :has_external_dependencies: true :urgency: :low @@ -1716,14 +2061,17 @@ :idempotent: :tags: [] - :name: experiments_record_conversion_event + :worker_name: Experiments::RecordConversionEventWorker :feature_category: :users :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: expire_build_instance_artifacts + :worker_name: ExpireBuildInstanceArtifactsWorker :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low @@ -1732,6 +2080,7 @@ :idempotent: :tags: [] - :name: export_csv + :worker_name: ExportCsvWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1740,6 +2089,7 @@ :idempotent: :tags: [] - :name: external_service_reactive_caching + :worker_name: ExternalServiceReactiveCachingWorker :feature_category: :not_owned :has_external_dependencies: true :urgency: :low @@ -1748,6 +2098,7 @@ :idempotent: :tags: [] - :name: file_hook + :worker_name: FileHookWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low @@ -1756,22 +2107,17 @@ :idempotent: :tags: [] - :name: flush_counter_increments + :worker_name: FlushCounterIncrementsWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] -- :name: git_garbage_collect - :feature_category: :gitaly - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 1 - :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: github_import_advance_stage + :worker_name: Gitlab::GithubImport::AdvanceStageWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1780,14 +2126,17 @@ :idempotent: :tags: [] - :name: gitlab_performance_bar_stats + :worker_name: GitlabPerformanceBarStatsWorker :feature_category: :metrics :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: gitlab_shell + :worker_name: GitlabShellWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :high @@ -1796,6 +2145,7 @@ :idempotent: :tags: [] - :name: group_destroy + :worker_name: GroupDestroyWorker :feature_category: :subgroups :has_external_dependencies: :urgency: :low @@ -1804,7 +2154,9 @@ :idempotent: :tags: - :requires_disk_io + - :exclude_from_kubernetes - :name: group_export + :worker_name: GroupExportWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1813,6 +2165,7 @@ :idempotent: :tags: [] - :name: group_import + :worker_name: GroupImportWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -1821,6 +2174,7 @@ :idempotent: :tags: [] - :name: import_issues_csv + :worker_name: ImportIssuesCsvWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1829,6 +2183,7 @@ :idempotent: true :tags: [] - :name: invalid_gpg_signature_update + :worker_name: InvalidGpgSignatureUpdateWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -1837,6 +2192,7 @@ :idempotent: :tags: [] - :name: irker + :worker_name: IrkerWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low @@ -1845,6 +2201,7 @@ :idempotent: :tags: [] - :name: issuable_export_csv + :worker_name: IssuableExportCsvWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low @@ -1852,7 +2209,26 @@ :weight: 1 :idempotent: :tags: [] +- :name: issuable_label_links_destroy + :worker_name: Issuable::LabelLinksDestroyWorker + :feature_category: :issue_tracking + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: issuables_clear_groups_issue_counter + :worker_name: Issuables::ClearGroupsIssueCounterWorker + :feature_category: :issue_tracking + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: issue_placement + :worker_name: IssuePlacementWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :high @@ -1861,14 +2237,17 @@ :idempotent: true :tags: [] - :name: issue_rebalancing + :worker_name: IssueRebalancingWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: mailers + :worker_name: ActionMailer::MailDeliveryJob :feature_category: :issue_tracking :has_external_dependencies: :urgency: low @@ -1877,22 +2256,26 @@ :idempotent: :tags: [] - :name: merge + :worker_name: MergeWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :high :resource_boundary: :unknown :weight: 5 - :idempotent: + :idempotent: true :tags: [] - :name: merge_request_cleanup_refs + :worker_name: MergeRequestCleanupRefsWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: merge_request_mergeability_check + :worker_name: MergeRequestMergeabilityCheckWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low @@ -1901,6 +2284,7 @@ :idempotent: true :tags: [] - :name: merge_requests_assignees_change + :worker_name: MergeRequests::AssigneesChangeWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :high @@ -1909,6 +2293,7 @@ :idempotent: true :tags: [] - :name: merge_requests_delete_source_branch + :worker_name: MergeRequests::DeleteSourceBranchWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :high @@ -1917,6 +2302,7 @@ :idempotent: true :tags: [] - :name: merge_requests_handle_assignees_change + :worker_name: MergeRequests::HandleAssigneesChangeWorker :feature_category: :code_review :has_external_dependencies: :urgency: :high @@ -1925,6 +2311,7 @@ :idempotent: true :tags: [] - :name: merge_requests_resolve_todos + :worker_name: MergeRequests::ResolveTodosWorker :feature_category: :code_review :has_external_dependencies: :urgency: :high @@ -1933,6 +2320,7 @@ :idempotent: true :tags: [] - :name: metrics_dashboard_prune_old_annotations + :worker_name: Metrics::Dashboard::PruneOldAnnotationsWorker :feature_category: :metrics :has_external_dependencies: :urgency: :low @@ -1941,14 +2329,17 @@ :idempotent: true :tags: [] - :name: metrics_dashboard_sync_dashboards + :worker_name: Metrics::Dashboard::SyncDashboardsWorker :feature_category: :metrics :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: migrate_external_diffs + :worker_name: MigrateExternalDiffsWorker :feature_category: :code_review :has_external_dependencies: :urgency: :low @@ -1957,6 +2348,7 @@ :idempotent: :tags: [] - :name: namespaceless_project_destroy + :worker_name: NamespacelessProjectDestroyWorker :feature_category: :authentication_and_authorization :has_external_dependencies: :urgency: :low @@ -1965,38 +2357,47 @@ :idempotent: :tags: [] - :name: namespaces_onboarding_issue_created + :worker_name: Namespaces::OnboardingIssueCreatedWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: namespaces_onboarding_pipeline_created + :worker_name: Namespaces::OnboardingPipelineCreatedWorker :feature_category: :subgroups :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: namespaces_onboarding_progress + :worker_name: Namespaces::OnboardingProgressWorker :feature_category: :product_analytics :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: namespaces_onboarding_user_added + :worker_name: Namespaces::OnboardingUserAddedWorker :feature_category: :users :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: new_issue + :worker_name: NewIssueWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :high @@ -2005,6 +2406,7 @@ :idempotent: :tags: [] - :name: new_merge_request + :worker_name: NewMergeRequestWorker :feature_category: :code_review :has_external_dependencies: :urgency: :high @@ -2013,6 +2415,7 @@ :idempotent: :tags: [] - :name: new_note + :worker_name: NewNoteWorker :feature_category: :issue_tracking :has_external_dependencies: :urgency: :high @@ -2021,14 +2424,17 @@ :idempotent: :tags: [] - :name: packages_composer_cache_update + :worker_name: Packages::Composer::CacheUpdateWorker :feature_category: :package_registry :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pages + :worker_name: PagesWorker :feature_category: :pages :has_external_dependencies: :urgency: :low @@ -2037,7 +2443,9 @@ :idempotent: :tags: - :requires_disk_io + - :exclude_from_kubernetes - :name: pages_domain_ssl_renewal + :worker_name: PagesDomainSslRenewalWorker :feature_category: :pages :has_external_dependencies: :urgency: :low @@ -2046,7 +2454,9 @@ :idempotent: :tags: - :requires_disk_io + - :exclude_from_kubernetes - :name: pages_domain_verification + :worker_name: PagesDomainVerificationWorker :feature_category: :pages :has_external_dependencies: :urgency: :low @@ -2055,31 +2465,39 @@ :idempotent: :tags: - :requires_disk_io + - :exclude_from_kubernetes - :name: pages_remove + :worker_name: PagesRemoveWorker :feature_category: :pages :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pages_transfer + :worker_name: PagesTransferWorker :feature_category: :pages :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: pages_update_configuration + :worker_name: PagesUpdateConfigurationWorker :feature_category: :pages :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: phabricator_import_import_tasks + :worker_name: Gitlab::PhabricatorImport::ImportTasksWorker :feature_category: :importers :has_external_dependencies: :urgency: :low @@ -2088,6 +2506,7 @@ :idempotent: :tags: [] - :name: post_receive + :worker_name: PostReceive :feature_category: :source_code_management :has_external_dependencies: :urgency: :high @@ -2096,6 +2515,7 @@ :idempotent: :tags: [] - :name: process_commit + :worker_name: ProcessCommitWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :high @@ -2104,6 +2524,7 @@ :idempotent: true :tags: [] - :name: project_cache + :worker_name: ProjectCacheWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :high @@ -2112,6 +2533,7 @@ :idempotent: true :tags: [] - :name: project_daily_statistics + :worker_name: ProjectDailyStatisticsWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2120,6 +2542,7 @@ :idempotent: :tags: [] - :name: project_destroy + :worker_name: ProjectDestroyWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2128,7 +2551,9 @@ :idempotent: :tags: - :requires_disk_io + - :exclude_from_kubernetes - :name: project_export + :worker_name: ProjectExportWorker :feature_category: :importers :has_external_dependencies: :urgency: :throttled @@ -2137,6 +2562,7 @@ :idempotent: :tags: [] - :name: project_schedule_bulk_repository_shard_moves + :worker_name: ProjectScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2145,6 +2571,7 @@ :idempotent: true :tags: [] - :name: project_service + :worker_name: ProjectServiceWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low @@ -2153,6 +2580,7 @@ :idempotent: :tags: [] - :name: project_update_repository_storage + :worker_name: ProjectUpdateRepositoryStorageWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2161,22 +2589,27 @@ :idempotent: true :tags: [] - :name: projects_git_garbage_collect + :worker_name: Projects::GitGarbageCollectWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: projects_post_creation + :worker_name: Projects::PostCreationWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: projects_schedule_bulk_repository_shard_moves + :worker_name: Projects::ScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2185,6 +2618,7 @@ :idempotent: true :tags: [] - :name: projects_update_repository_storage + :worker_name: Projects::UpdateRepositoryStorageWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2193,6 +2627,7 @@ :idempotent: true :tags: [] - :name: prometheus_create_default_alerts + :worker_name: Prometheus::CreateDefaultAlertsWorker :feature_category: :incident_management :has_external_dependencies: :urgency: :high @@ -2201,6 +2636,7 @@ :idempotent: true :tags: [] - :name: propagate_integration + :worker_name: PropagateIntegrationWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low @@ -2209,38 +2645,47 @@ :idempotent: true :tags: [] - :name: propagate_integration_group + :worker_name: PropagateIntegrationGroupWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: propagate_integration_inherit + :worker_name: PropagateIntegrationInheritWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: propagate_integration_inherit_descendant + :worker_name: PropagateIntegrationInheritDescendantWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: propagate_integration_project + :worker_name: PropagateIntegrationProjectWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: propagate_service_template + :worker_name: PropagateServiceTemplateWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low @@ -2249,6 +2694,7 @@ :idempotent: :tags: [] - :name: reactive_caching + :worker_name: ReactiveCachingWorker :feature_category: :not_owned :has_external_dependencies: :urgency: :low @@ -2257,6 +2703,7 @@ :idempotent: :tags: [] - :name: rebase + :worker_name: RebaseWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2265,14 +2712,17 @@ :idempotent: :tags: [] - :name: releases_create_evidence + :worker_name: Releases::CreateEvidenceWorker :feature_category: :release_evidence :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: remote_mirror_notification + :worker_name: RemoteMirrorNotificationWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2281,6 +2731,7 @@ :idempotent: :tags: [] - :name: repository_cleanup + :worker_name: RepositoryCleanupWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2289,6 +2740,7 @@ :idempotent: :tags: [] - :name: repository_fork + :worker_name: RepositoryForkWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2297,6 +2749,7 @@ :idempotent: :tags: [] - :name: repository_import + :worker_name: RepositoryImportWorker :feature_category: :importers :has_external_dependencies: true :urgency: :low @@ -2305,6 +2758,7 @@ :idempotent: :tags: [] - :name: repository_remove_remote + :worker_name: RepositoryRemoveRemoteWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2313,6 +2767,7 @@ :idempotent: :tags: [] - :name: repository_update_remote_mirror + :worker_name: RepositoryUpdateRemoteMirrorWorker :feature_category: :source_code_management :has_external_dependencies: true :urgency: :low @@ -2321,6 +2776,7 @@ :idempotent: true :tags: [] - :name: self_monitoring_project_create + :worker_name: SelfMonitoringProjectCreateWorker :feature_category: :metrics :has_external_dependencies: :urgency: :low @@ -2329,6 +2785,7 @@ :idempotent: :tags: [] - :name: self_monitoring_project_delete + :worker_name: SelfMonitoringProjectDeleteWorker :feature_category: :metrics :has_external_dependencies: :urgency: :low @@ -2337,7 +2794,8 @@ :idempotent: :tags: [] - :name: service_desk_email_receiver - :feature_category: :issue_tracking + :worker_name: ServiceDeskEmailReceiverWorker + :feature_category: :service_desk :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -2345,6 +2803,7 @@ :idempotent: :tags: [] - :name: snippet_schedule_bulk_repository_shard_moves + :worker_name: SnippetScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2353,6 +2812,7 @@ :idempotent: true :tags: [] - :name: snippet_update_repository_storage + :worker_name: SnippetUpdateRepositoryStorageWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2361,6 +2821,7 @@ :idempotent: true :tags: [] - :name: snippets_schedule_bulk_repository_shard_moves + :worker_name: Snippets::ScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2369,6 +2830,7 @@ :idempotent: true :tags: [] - :name: snippets_update_repository_storage + :worker_name: Snippets::UpdateRepositoryStorageWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :throttled @@ -2377,6 +2839,7 @@ :idempotent: true :tags: [] - :name: system_hook_push + :worker_name: SystemHookPushWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2385,6 +2848,7 @@ :idempotent: :tags: [] - :name: update_external_pull_requests + :worker_name: UpdateExternalPullRequestsWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2393,6 +2857,7 @@ :idempotent: :tags: [] - :name: update_highest_role + :worker_name: UpdateHighestRoleWorker :feature_category: :utilization :has_external_dependencies: :urgency: :high @@ -2401,6 +2866,7 @@ :idempotent: true :tags: [] - :name: update_merge_requests + :worker_name: UpdateMergeRequestsWorker :feature_category: :code_review :has_external_dependencies: :urgency: :high @@ -2409,6 +2875,7 @@ :idempotent: :tags: [] - :name: update_project_statistics + :worker_name: UpdateProjectStatisticsWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low @@ -2417,6 +2884,7 @@ :idempotent: :tags: [] - :name: upload_checksum + :worker_name: UploadChecksumWorker :feature_category: :geo_replication :has_external_dependencies: :urgency: :low @@ -2424,7 +2892,18 @@ :weight: 1 :idempotent: :tags: [] +- :name: users_update_open_issue_count + :worker_name: Users::UpdateOpenIssueCountWorker + :feature_category: :users + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: + - :exclude_from_kubernetes - :name: web_hook + :worker_name: WebHookWorker :feature_category: :integrations :has_external_dependencies: true :urgency: :low @@ -2433,22 +2912,27 @@ :idempotent: :tags: [] - :name: web_hooks_destroy + :worker_name: WebHooks::DestroyWorker :feature_category: :integrations :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: wikis_git_garbage_collect + :worker_name: Wikis::GitGarbageCollectWorker :feature_category: :gitaly :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :exclude_from_kubernetes - :name: x509_certificate_revoke + :worker_name: X509CertificateRevokeWorker :feature_category: :source_code_management :has_external_dependencies: :urgency: :low diff --git a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb index 3ec92bc7635..083c01b166d 100644 --- a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb +++ b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb @@ -5,9 +5,12 @@ module Analytics # This worker will be removed in 14.0 class CountJobTriggerWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :devops_reports + tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/analytics/instance_statistics/counter_job_worker.rb b/app/workers/analytics/instance_statistics/counter_job_worker.rb index 4beed8a3e2f..a4dda45ff72 100644 --- a/app/workers/analytics/instance_statistics/counter_job_worker.rb +++ b/app/workers/analytics/instance_statistics/counter_job_worker.rb @@ -6,8 +6,11 @@ module Analytics class CounterJobWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :devops_reports urgency :low + tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/analytics/usage_trends/count_job_trigger_worker.rb b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb index 37f5c19d64c..f2d4404a964 100644 --- a/app/workers/analytics/usage_trends/count_job_trigger_worker.rb +++ b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb @@ -5,11 +5,14 @@ module Analytics class CountJobTriggerWorker extend ::Gitlab::Utils::Override include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext DEFAULT_DELAY = 3.minutes.freeze feature_category :devops_reports + tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/analytics/usage_trends/counter_job_worker.rb b/app/workers/analytics/usage_trends/counter_job_worker.rb index 275c6ac2de2..f4dc497d25f 100644 --- a/app/workers/analytics/usage_trends/counter_job_worker.rb +++ b/app/workers/analytics/usage_trends/counter_job_worker.rb @@ -6,8 +6,11 @@ module Analytics extend ::Gitlab::Utils::Override include ApplicationWorker + sidekiq_options retry: 3 + feature_category :devops_reports urgency :low + tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/approve_blocked_pending_approval_users_worker.rb b/app/workers/approve_blocked_pending_approval_users_worker.rb index 8ca61d68bfd..ff72aaad3ce 100644 --- a/app/workers/approve_blocked_pending_approval_users_worker.rb +++ b/app/workers/approve_blocked_pending_approval_users_worker.rb @@ -3,9 +3,12 @@ class ApproveBlockedPendingApprovalUsersWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! feature_category :users + tags :exclude_from_kubernetes def perform(current_user_id) current_user = User.find(current_user_id) diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb index 3ddb5686bf2..629526ec17c 100644 --- a/app/workers/archive_trace_worker.rb +++ b/app/workers/archive_trace_worker.rb @@ -2,6 +2,8 @@ class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineBackgroundQueue # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/authorized_keys_worker.rb b/app/workers/authorized_keys_worker.rb index ab0e7fc4921..953f493ea2c 100644 --- a/app/workers/authorized_keys_worker.rb +++ b/app/workers/authorized_keys_worker.rb @@ -3,6 +3,8 @@ class AuthorizedKeysWorker include ApplicationWorker + sidekiq_options retry: 3 + PERMITTED_ACTIONS = %w[add_key remove_key].freeze feature_category :source_code_management diff --git a/app/workers/authorized_project_update/periodic_recalculate_worker.rb b/app/workers/authorized_project_update/periodic_recalculate_worker.rb index 78ffdbca4d6..2f6a9c42c0c 100644 --- a/app/workers/authorized_project_update/periodic_recalculate_worker.rb +++ b/app/workers/authorized_project_update/periodic_recalculate_worker.rb @@ -3,6 +3,8 @@ module AuthorizedProjectUpdate class PeriodicRecalculateWorker include ApplicationWorker + + sidekiq_options retry: 3 # This worker does not perform work scoped to a context include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/authorized_project_update/project_create_worker.rb b/app/workers/authorized_project_update/project_create_worker.rb index 651849b57ec..52b740b4efe 100644 --- a/app/workers/authorized_project_update/project_create_worker.rb +++ b/app/workers/authorized_project_update/project_create_worker.rb @@ -4,6 +4,8 @@ module AuthorizedProjectUpdate class ProjectCreateWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :authentication_and_authorization urgency :low queue_namespace :authorized_project_update diff --git a/app/workers/authorized_project_update/project_group_link_create_worker.rb b/app/workers/authorized_project_update/project_group_link_create_worker.rb index dd24a9602bb..d887a2ce25f 100644 --- a/app/workers/authorized_project_update/project_group_link_create_worker.rb +++ b/app/workers/authorized_project_update/project_group_link_create_worker.rb @@ -4,6 +4,8 @@ module AuthorizedProjectUpdate class ProjectGroupLinkCreateWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :authentication_and_authorization urgency :low queue_namespace :authorized_project_update diff --git a/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb b/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb index 6635c322ab8..2e4e2dd3232 100644 --- a/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb +++ b/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb @@ -15,6 +15,8 @@ module AuthorizedProjectUpdate include ApplicationWorker + sidekiq_options retry: 3 + feature_category :authentication_and_authorization urgency :low queue_namespace :authorized_project_update @@ -22,7 +24,7 @@ module AuthorizedProjectUpdate # `data_consistency :delayed` and not `idempotent!` # See https://gitlab.com/gitlab-org/gitlab/-/issues/325291 deduplicate :until_executing, including_scheduled: true - data_consistency :delayed, feature_flag: :periodic_project_authorization_update_via_replica + data_consistency :delayed, feature_flag: :delayed_consistency_for_user_refresh_over_range_worker def perform(start_user_id, end_user_id) if Feature.enabled?(:periodic_project_authorization_update_via_replica) @@ -30,12 +32,17 @@ module AuthorizedProjectUpdate enqueue_project_authorizations_refresh(user) if project_authorizations_needs_refresh?(user) end else + use_primary_database AuthorizedProjectUpdate::RecalculateForUserRangeService.new(start_user_id, end_user_id).execute end end private + def use_primary_database + # no-op in CE, overriden in EE + end + def project_authorizations_needs_refresh?(user) AuthorizedProjectUpdate::FindRecordsDueForRefreshService.new(user).needs_refresh? end @@ -47,3 +54,5 @@ module AuthorizedProjectUpdate end end end + +AuthorizedProjectUpdate::UserRefreshOverUserRangeWorker.prepend_mod_with('AuthorizedProjectUpdate::UserRefreshOverUserRangeWorker') diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index 6e07d6d0f71..a1068117e59 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -2,6 +2,8 @@ class AuthorizedProjectsWorker include ApplicationWorker + + sidekiq_options retry: 3 prepend WaitableWorker feature_category :authentication_and_authorization diff --git a/app/workers/auto_devops/disable_worker.rb b/app/workers/auto_devops/disable_worker.rb index bae08cf9e18..43377382e82 100644 --- a/app/workers/auto_devops/disable_worker.rb +++ b/app/workers/auto_devops/disable_worker.rb @@ -3,6 +3,8 @@ module AutoDevops class DisableWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include AutoDevopsQueue def perform(pipeline_id) diff --git a/app/workers/auto_merge_process_worker.rb b/app/workers/auto_merge_process_worker.rb index 2599c76c900..dda0e970834 100644 --- a/app/workers/auto_merge_process_worker.rb +++ b/app/workers/auto_merge_process_worker.rb @@ -3,6 +3,8 @@ class AutoMergeProcessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :auto_merge feature_category :continuous_delivery worker_resource_boundary :cpu diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index 70c4ad53726..6b1f10f75b8 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -3,6 +3,8 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :database urgency :throttled loggable_arguments 0, 1 diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index aeda8d113ac..a3eaacec8a2 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -2,6 +2,8 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing @@ -34,7 +36,6 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker # We execute these async as these are independent operations. BuildHooksWorker.perform_async(build.id) - ExpirePipelineCacheWorker.perform_async(build.pipeline_id) ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat? if build.failed? @@ -57,4 +58,4 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker end end -BuildFinishedWorker.prepend_if_ee('EE::BuildFinishedWorker') +BuildFinishedWorker.prepend_mod_with('BuildFinishedWorker') diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb index 5e05063f058..be79d6b2afb 100644 --- a/app/workers/build_hooks_worker.rb +++ b/app/workers/build_hooks_worker.rb @@ -2,6 +2,8 @@ class BuildHooksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_hooks @@ -9,6 +11,16 @@ class BuildHooksWorker # rubocop:disable Scalability/IdempotentWorker urgency :high data_consistency :delayed, feature_flag: :load_balancing_for_build_hooks_worker + DATA_CONSISTENCY_DELAY = 3 + + def self.perform_async(*args) + if Feature.enabled?(:delayed_perform_for_build_hooks_worker, default_enabled: :yaml) + perform_in(DATA_CONSISTENCY_DELAY.seconds, *args) + else + super + end + end + # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) Ci::Build.includes({ runner: :tags }) diff --git a/app/workers/build_queue_worker.rb b/app/workers/build_queue_worker.rb index b71afbbeb8f..e9bb2d88a81 100644 --- a/app/workers/build_queue_worker.rb +++ b/app/workers/build_queue_worker.rb @@ -2,6 +2,8 @@ class BuildQueueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb index e4a2dd500cc..531e7e5a5fe 100644 --- a/app/workers/build_success_worker.rb +++ b/app/workers/build_success_worker.rb @@ -2,6 +2,8 @@ class BuildSuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/bulk_import_worker.rb b/app/workers/bulk_import_worker.rb index b4b9d9b05c1..8ad31c68374 100644 --- a/app/workers/bulk_import_worker.rb +++ b/app/workers/bulk_import_worker.rb @@ -4,6 +4,7 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :importers + tags :exclude_from_kubernetes sidekiq_options retry: false, dead: false @@ -23,13 +24,14 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker created_entities.first(next_batch_size).each do |entity| create_pipeline_tracker_for(entity) + BulkImports::ExportRequestWorker.perform_async(entity.id) BulkImports::EntityWorker.perform_async(entity.id) entity.start! end re_enqueue - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, bulk_import_id: @bulk_import&.id) @bulk_import&.fail_op diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb index 7f173b738cf..e7fce112ee1 100644 --- a/app/workers/bulk_imports/entity_worker.rb +++ b/app/workers/bulk_imports/entity_worker.rb @@ -5,6 +5,7 @@ module BulkImports include ApplicationWorker feature_category :importers + tags :exclude_from_kubernetes sidekiq_options retry: false, dead: false @@ -26,7 +27,7 @@ module BulkImports entity_id ) end - rescue => e + rescue StandardError => e logger.error( worker: self.class.name, entity_id: entity_id, diff --git a/app/workers/bulk_imports/export_request_worker.rb b/app/workers/bulk_imports/export_request_worker.rb new file mode 100644 index 00000000000..cccc24d3bdc --- /dev/null +++ b/app/workers/bulk_imports/export_request_worker.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module BulkImports + class ExportRequestWorker + include ApplicationWorker + + idempotent! + worker_has_external_dependencies! + feature_category :importers + + GROUP_EXPORTED_URL_PATH = "/groups/%s/export_relations" + + def perform(entity_id) + entity = BulkImports::Entity.find(entity_id) + + request_export(entity) + end + + private + + def request_export(entity) + http_client(entity.bulk_import.configuration) + .post(GROUP_EXPORTED_URL_PATH % entity.encoded_source_full_path) + end + + def http_client(configuration) + @client ||= Clients::Http.new( + uri: configuration.url, + token: configuration.access_token + ) + end + end +end diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb index a6de3c36205..256301bf097 100644 --- a/app/workers/bulk_imports/pipeline_worker.rb +++ b/app/workers/bulk_imports/pipeline_worker.rb @@ -5,6 +5,7 @@ module BulkImports include ApplicationWorker feature_category :importers + tags :exclude_from_kubernetes sidekiq_options retry: false, dead: false @@ -46,7 +47,7 @@ module BulkImports pipeline_tracker.pipeline_class.new(context).run pipeline_tracker.finish! - rescue => e + rescue StandardError => e pipeline_tracker.fail_op! logger.error( diff --git a/app/workers/bulk_imports/relation_export_worker.rb b/app/workers/bulk_imports/relation_export_worker.rb new file mode 100644 index 00000000000..9d9449e3a1b --- /dev/null +++ b/app/workers/bulk_imports/relation_export_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module BulkImports + class RelationExportWorker + include ApplicationWorker + include ExceptionBacktrace + + idempotent! + loggable_arguments 2, 3 + feature_category :importers + tags :exclude_from_kubernetes + sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION + + def perform(user_id, portable_id, portable_class, relation) + user = User.find(user_id) + portable = portable(portable_id, portable_class) + + RelationExportService.new(user, portable, relation, jid).execute + end + + private + + def portable(portable_id, portable_class) + portable_class.classify.constantize.find(portable_id) + end + end +end diff --git a/app/workers/chaos/cpu_spin_worker.rb b/app/workers/chaos/cpu_spin_worker.rb index 0b565e0d49c..f8900abc764 100644 --- a/app/workers/chaos/cpu_spin_worker.rb +++ b/app/workers/chaos/cpu_spin_worker.rb @@ -3,6 +3,8 @@ module Chaos class CpuSpinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ChaosQueue def perform(duration_s) diff --git a/app/workers/chaos/db_spin_worker.rb b/app/workers/chaos/db_spin_worker.rb index 099660d440c..9b5d06414a9 100644 --- a/app/workers/chaos/db_spin_worker.rb +++ b/app/workers/chaos/db_spin_worker.rb @@ -3,6 +3,8 @@ module Chaos class DbSpinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ChaosQueue def perform(duration_s, interval_s) diff --git a/app/workers/chaos/leak_mem_worker.rb b/app/workers/chaos/leak_mem_worker.rb index b77d1a20541..788009962db 100644 --- a/app/workers/chaos/leak_mem_worker.rb +++ b/app/workers/chaos/leak_mem_worker.rb @@ -3,6 +3,8 @@ module Chaos class LeakMemWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ChaosQueue def perform(memory_mb, duration_s) diff --git a/app/workers/chaos/sleep_worker.rb b/app/workers/chaos/sleep_worker.rb index 6887258e961..b9ff5546384 100644 --- a/app/workers/chaos/sleep_worker.rb +++ b/app/workers/chaos/sleep_worker.rb @@ -3,6 +3,8 @@ module Chaos class SleepWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ChaosQueue def perform(duration_s) diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb index 0171c1d482d..c748bc33ada 100644 --- a/app/workers/ci/archive_traces_cron_worker.rb +++ b/app/workers/ci/archive_traces_cron_worker.rb @@ -3,6 +3,8 @@ module Ci class ArchiveTracesCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :continuous_integration diff --git a/app/workers/ci/build_prepare_worker.rb b/app/workers/ci/build_prepare_worker.rb index 7f640633070..f30e9d3b885 100644 --- a/app/workers/ci/build_prepare_worker.rb +++ b/app/workers/ci/build_prepare_worker.rb @@ -3,6 +3,8 @@ module Ci class BuildPrepareWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index 9231b40978d..570f5f28c3d 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -3,6 +3,8 @@ module Ci class BuildScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/ci/build_trace_chunk_flush_worker.rb b/app/workers/ci/build_trace_chunk_flush_worker.rb index a63b12c0d03..1e0da73e08d 100644 --- a/app/workers/ci/build_trace_chunk_flush_worker.rb +++ b/app/workers/ci/build_trace_chunk_flush_worker.rb @@ -3,6 +3,8 @@ module Ci class BuildTraceChunkFlushWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineBackgroundQueue deduplicate :until_executed diff --git a/app/workers/ci/create_cross_project_pipeline_worker.rb b/app/workers/ci/create_cross_project_pipeline_worker.rb index 679574d9f60..4881ee12e5c 100644 --- a/app/workers/ci/create_cross_project_pipeline_worker.rb +++ b/app/workers/ci/create_cross_project_pipeline_worker.rb @@ -5,6 +5,7 @@ module Ci include ::ApplicationWorker include ::PipelineQueue + sidekiq_options retry: 3 worker_resource_boundary :cpu def perform(bridge_id) diff --git a/app/workers/ci/daily_build_group_report_results_worker.rb b/app/workers/ci/daily_build_group_report_results_worker.rb index 687cadc6366..b38bef3bcf8 100644 --- a/app/workers/ci/daily_build_group_report_results_worker.rb +++ b/app/workers/ci/daily_build_group_report_results_worker.rb @@ -3,6 +3,8 @@ module Ci class DailyBuildGroupReportResultsWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineBackgroundQueue feature_category :code_testing diff --git a/app/workers/ci/delete_objects_worker.rb b/app/workers/ci/delete_objects_worker.rb index d845ad61358..ff020a3b048 100644 --- a/app/workers/ci/delete_objects_worker.rb +++ b/app/workers/ci/delete_objects_worker.rb @@ -3,9 +3,12 @@ module Ci class DeleteObjectsWorker include ApplicationWorker + + sidekiq_options retry: 3 include LimitedCapacity::Worker feature_category :continuous_integration + tags :exclude_from_kubernetes idempotent! def perform_work(*args) diff --git a/app/workers/ci/delete_unit_tests_worker.rb b/app/workers/ci/delete_unit_tests_worker.rb new file mode 100644 index 00000000000..ddfc70c43d4 --- /dev/null +++ b/app/workers/ci/delete_unit_tests_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class DeleteUnitTestsWorker + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker does not perform work scoped to a context + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :continuous_integration + idempotent! + + def perform + Ci::DeleteUnitTestsService.new.execute + end + end +end diff --git a/app/workers/ci/drop_pipeline_worker.rb b/app/workers/ci/drop_pipeline_worker.rb index d19157a47e8..bc158433228 100644 --- a/app/workers/ci/drop_pipeline_worker.rb +++ b/app/workers/ci/drop_pipeline_worker.rb @@ -3,8 +3,12 @@ module Ci class DropPipelineWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue + tags :exclude_from_kubernetes + idempotent! def perform(pipeline_id, failure_reason) diff --git a/app/workers/ci/initial_pipeline_process_worker.rb b/app/workers/ci/initial_pipeline_process_worker.rb index f59726c87fb..4dace43298d 100644 --- a/app/workers/ci/initial_pipeline_process_worker.rb +++ b/app/workers/ci/initial_pipeline_process_worker.rb @@ -3,6 +3,8 @@ module Ci class InitialPipelineProcessWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb b/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb index d5e097dc2b5..bd061b5f988 100644 --- a/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb +++ b/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb @@ -3,9 +3,12 @@ module Ci module MergeRequests class AddTodoWhenBuildFailsWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue urgency :low + tags :exclude_from_kubernetes idempotent! def perform(job_id) @@ -14,7 +17,7 @@ module Ci return unless job && project - ::MergeRequests::AddTodoWhenBuildFailsService.new(job.project, nil).execute(job) + ::MergeRequests::AddTodoWhenBuildFailsService.new(project: job.project).execute(job) end end end diff --git a/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb b/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb index 4de56f54f44..dd7bfff4eb1 100644 --- a/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb +++ b/app/workers/ci/pipeline_artifacts/coverage_report_worker.rb @@ -4,9 +4,12 @@ module Ci module PipelineArtifacts class CoverageReportWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineBackgroundQueue feature_category :code_testing + tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb index 810106e8d9c..558153c69b2 100644 --- a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb +++ b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb @@ -5,14 +5,17 @@ module Ci class CreateQualityReportWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :pipeline_background feature_category :code_testing + tags :exclude_from_kubernetes idempotent! def perform(pipeline_id) Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline| - Ci::PipelineArtifacts::CreateCodeQualityMrDiffReportService.new.execute(pipeline) + Ci::PipelineArtifacts::CreateCodeQualityMrDiffReportService.new(pipeline).execute end end end diff --git a/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb b/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb index fff979d95a9..004c1d444a2 100644 --- a/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb +++ b/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb @@ -4,6 +4,8 @@ module Ci module PipelineArtifacts class ExpireArtifactsWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue @@ -12,6 +14,7 @@ module Ci deduplicate :until_executed, including_scheduled: true idempotent! feature_category :continuous_integration + tags :exclude_from_kubernetes def perform service = ::Ci::PipelineArtifacts::DestroyAllExpiredService.new diff --git a/app/workers/ci/pipeline_bridge_status_worker.rb b/app/workers/ci/pipeline_bridge_status_worker.rb index 3f92f4561e0..3630331b41d 100644 --- a/app/workers/ci/pipeline_bridge_status_worker.rb +++ b/app/workers/ci/pipeline_bridge_status_worker.rb @@ -5,6 +5,7 @@ module Ci include ::ApplicationWorker include ::PipelineQueue + sidekiq_options retry: 3 urgency :high worker_resource_boundary :cpu diff --git a/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb b/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb index bc31876aa1d..b0921f6e10b 100644 --- a/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb +++ b/app/workers/ci/pipeline_success_unlock_artifacts_worker.rb @@ -3,6 +3,8 @@ module Ci class PipelineSuccessUnlockArtifactsWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineBackgroundQueue idempotent! diff --git a/app/workers/ci/ref_delete_unlock_artifacts_worker.rb b/app/workers/ci/ref_delete_unlock_artifacts_worker.rb index aaa77efbb74..d20c501100e 100644 --- a/app/workers/ci/ref_delete_unlock_artifacts_worker.rb +++ b/app/workers/ci/ref_delete_unlock_artifacts_worker.rb @@ -3,6 +3,8 @@ module Ci class RefDeleteUnlockArtifactsWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineBackgroundQueue idempotent! diff --git a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb index 8063e34a1b8..15ed89fd00e 100644 --- a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb +++ b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb @@ -4,6 +4,8 @@ module Ci module ResourceGroups class AssignResourceFromResourceGroupWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/ci/retry_pipeline_worker.rb b/app/workers/ci/retry_pipeline_worker.rb new file mode 100644 index 00000000000..7a1906b3ef9 --- /dev/null +++ b/app/workers/ci/retry_pipeline_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Ci + class RetryPipelineWorker # rubocop:disable Scalability/IdempotentWorker + include ::ApplicationWorker + include ::PipelineQueue + + urgency :high + worker_resource_boundary :cpu + + def perform(pipeline_id, user_id) + ::Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline| + ::User.find_by_id(user_id).try do |user| + pipeline.retry_failed(user) + end + end + end + end +end diff --git a/app/workers/ci/schedule_delete_objects_cron_worker.rb b/app/workers/ci/schedule_delete_objects_cron_worker.rb index fa0b15deb56..6489665fafd 100644 --- a/app/workers/ci/schedule_delete_objects_cron_worker.rb +++ b/app/workers/ci/schedule_delete_objects_cron_worker.rb @@ -3,12 +3,15 @@ module Ci class ScheduleDeleteObjectsCronWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue # rubocop:enable Scalability/CronWorkerContext feature_category :continuous_integration + tags :exclude_from_kubernetes idempotent! def perform(*args) diff --git a/app/workers/ci/test_failure_history_worker.rb b/app/workers/ci/test_failure_history_worker.rb index e1562cb3836..3937f720788 100644 --- a/app/workers/ci/test_failure_history_worker.rb +++ b/app/workers/ci/test_failure_history_worker.rb @@ -3,8 +3,12 @@ module Ci class TestFailureHistoryWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineBackgroundQueue + tags :exclude_from_kubernetes + idempotent! def perform(pipeline_id) diff --git a/app/workers/ci_platform_metrics_update_cron_worker.rb b/app/workers/ci_platform_metrics_update_cron_worker.rb index ec1fc26fad3..05af0a0a73b 100644 --- a/app/workers/ci_platform_metrics_update_cron_worker.rb +++ b/app/workers/ci_platform_metrics_update_cron_worker.rb @@ -3,6 +3,8 @@ class CiPlatformMetricsUpdateCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + # This worker does not perform work scoped to a context include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb index 1cac2858156..a8de8efbce6 100644 --- a/app/workers/cleanup_container_repository_worker.rb +++ b/app/workers/cleanup_container_repository_worker.rb @@ -3,6 +3,8 @@ class CleanupContainerRepositoryWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :container_repository feature_category :container_registry urgency :low diff --git a/app/workers/cluster_configure_istio_worker.rb b/app/workers/cluster_configure_istio_worker.rb index ec6bdfbd6b6..07c032da838 100644 --- a/app/workers/cluster_configure_istio_worker.rb +++ b/app/workers/cluster_configure_istio_worker.rb @@ -2,6 +2,8 @@ class ClusterConfigureIstioWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue worker_has_external_dependencies! diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb index f3da4d5c4bb..71374de19f5 100644 --- a/app/workers/cluster_install_app_worker.rb +++ b/app/workers/cluster_install_app_worker.rb @@ -2,6 +2,8 @@ class ClusterInstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_patch_app_worker.rb b/app/workers/cluster_patch_app_worker.rb index b0393809802..674a565f7f7 100644 --- a/app/workers/cluster_patch_app_worker.rb +++ b/app/workers/cluster_patch_app_worker.rb @@ -2,6 +2,8 @@ class ClusterPatchAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb index cb750f3021e..142ad84f746 100644 --- a/app/workers/cluster_provision_worker.rb +++ b/app/workers/cluster_provision_worker.rb @@ -2,6 +2,8 @@ class ClusterProvisionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue worker_has_external_dependencies! diff --git a/app/workers/cluster_update_app_worker.rb b/app/workers/cluster_update_app_worker.rb index 29feb813043..dc57a1a90d9 100644 --- a/app/workers/cluster_update_app_worker.rb +++ b/app/workers/cluster_update_app_worker.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Deprecated, to be removed in %14.0 as part of https://gitlab.com/groups/gitlab-org/-/epics/4280 class ClusterUpdateAppWorker # rubocop:disable Scalability/IdempotentWorker UpdateAlreadyInProgressError = Class.new(StandardError) @@ -35,6 +36,7 @@ class ClusterUpdateAppWorker # rubocop:disable Scalability/IdempotentWorker # rubocop: enable CodeReuse/ActiveRecord def update_prometheus(app, scheduled_time, project) + return unless app.managed_prometheus? return if app.updated_since?(scheduled_time) return if app.update_in_progress? diff --git a/app/workers/cluster_upgrade_app_worker.rb b/app/workers/cluster_upgrade_app_worker.rb index d4650ab3a85..909ada2044f 100644 --- a/app/workers/cluster_upgrade_app_worker.rb +++ b/app/workers/cluster_upgrade_app_worker.rb @@ -2,6 +2,8 @@ class ClusterUpgradeAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb index 4bc29807ea4..19e33cd17b0 100644 --- a/app/workers/cluster_wait_for_app_installation_worker.rb +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -2,6 +2,8 @@ class ClusterWaitForAppInstallationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_app_update_worker.rb b/app/workers/cluster_wait_for_app_update_worker.rb index c0a11eb93a7..185959884a1 100644 --- a/app/workers/cluster_wait_for_app_update_worker.rb +++ b/app/workers/cluster_wait_for_app_update_worker.rb @@ -2,6 +2,8 @@ class ClusterWaitForAppUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb index fa46135d279..4a010c749a2 100644 --- a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb +++ b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb @@ -2,6 +2,8 @@ class ClusterWaitForIngressIpAddressWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/clusters/applications/activate_service_worker.rb b/app/workers/clusters/applications/activate_service_worker.rb index c92f978a7d2..d4d0ae96e03 100644 --- a/app/workers/clusters/applications/activate_service_worker.rb +++ b/app/workers/clusters/applications/activate_service_worker.rb @@ -4,6 +4,8 @@ module Clusters module Applications class ActivateServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue loggable_arguments 1 diff --git a/app/workers/clusters/applications/check_prometheus_health_worker.rb b/app/workers/clusters/applications/check_prometheus_health_worker.rb index cf9534c9a78..4db7314cbc0 100644 --- a/app/workers/clusters/applications/check_prometheus_health_worker.rb +++ b/app/workers/clusters/applications/check_prometheus_health_worker.rb @@ -4,6 +4,8 @@ module Clusters module Applications class CheckPrometheusHealthWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/clusters/applications/deactivate_service_worker.rb b/app/workers/clusters/applications/deactivate_service_worker.rb index 4d103bb0edc..935b455a4fc 100644 --- a/app/workers/clusters/applications/deactivate_service_worker.rb +++ b/app/workers/clusters/applications/deactivate_service_worker.rb @@ -4,6 +4,8 @@ module Clusters module Applications class DeactivateServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue loggable_arguments 1 diff --git a/app/workers/clusters/applications/uninstall_worker.rb b/app/workers/clusters/applications/uninstall_worker.rb index a9307931b59..3a4564ca7ab 100644 --- a/app/workers/clusters/applications/uninstall_worker.rb +++ b/app/workers/clusters/applications/uninstall_worker.rb @@ -4,6 +4,8 @@ module Clusters module Applications class UninstallWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb index dc842788374..18801ad7e64 100644 --- a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb +++ b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb @@ -4,6 +4,8 @@ module Clusters module Applications class WaitForUninstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue include ClusterApplications diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index 0de26e27631..843be4896a3 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -16,6 +16,7 @@ module ApplicationWorker included do set_queue + after_set_class_attribute { set_queue } def structured_payload(payload = {}) context = Gitlab::ApplicationContext.current.merge( @@ -47,22 +48,14 @@ module ApplicationWorker class_methods do def inherited(subclass) subclass.set_queue + subclass.after_set_class_attribute { subclass.set_queue } end def set_queue - queue_name = [queue_namespace, base_queue_name].compact.join(':') - + queue_name = ::Gitlab::SidekiqConfig::WorkerRouter.global.route(self) sidekiq_options queue: queue_name # rubocop:disable Cop/SidekiqOptionsQueue end - def base_queue_name - name - .sub(/\AGitlab::/, '') - .sub(/Worker\z/, '') - .underscore - .tr('/', '_') - end - def queue_namespace(new_namespace = nil) if new_namespace sidekiq_options queue_namespace: new_namespace diff --git a/app/workers/concerns/chaos_queue.rb b/app/workers/concerns/chaos_queue.rb index a9c557f0175..2ccd55157c6 100644 --- a/app/workers/concerns/chaos_queue.rb +++ b/app/workers/concerns/chaos_queue.rb @@ -6,5 +6,6 @@ module ChaosQueue included do queue_namespace :chaos feature_category_not_owned! + tags :exclude_from_gitlab_com end end diff --git a/app/workers/concerns/git_garbage_collect_methods.rb b/app/workers/concerns/git_garbage_collect_methods.rb index 17a80d1ddb3..c46deeb716f 100644 --- a/app/workers/concerns/git_garbage_collect_methods.rb +++ b/app/workers/concerns/git_garbage_collect_methods.rb @@ -97,10 +97,10 @@ module GitGarbageCollectMethods end rescue GRPC::NotFound => e Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found") - raise Gitlab::Git::Repository::NoRepository.new(e) + raise Gitlab::Git::Repository::NoRepository, e rescue GRPC::BadStatus => e Gitlab::GitLogger.error("#{__method__} failed:\n#{e}") - raise Gitlab::Git::CommandError.new(e) + raise Gitlab::Git::CommandError, e end def get_gitaly_client(task, repository) diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index 575cd4862b0..6ebf7c7c263 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -9,6 +9,8 @@ module Gitlab included do include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include ReschedulingMethods include Gitlab::NotifyUponDeath @@ -25,15 +27,19 @@ module Gitlab # client - An instance of `Gitlab::GithubImport::Client` # hash - A Hash containing the details of the object to import. def import(project, client, hash) + object = representation_class.from_json_hash(hash) + + # To better express in the logs what object is being imported. + self.github_id = object.attributes.fetch(:github_id) + info(project.id, message: 'starting importer') - object = representation_class.from_json_hash(hash) importer_class.new(object, project, client).execute counter.increment info(project.id, message: 'importer finished') - rescue => e - error(project.id, e) + rescue StandardError => e + error(project.id, e, hash) end def counter @@ -63,16 +69,19 @@ module Gitlab private + attr_accessor :github_id + def info(project_id, extra = {}) logger.info(log_attributes(project_id, extra)) end - def error(project_id, exception) + def error(project_id, exception, data = {}) logger.error( log_attributes( project_id, message: 'importer failed', - 'error.message': exception.message + 'error.message': exception.message, + 'github.data': data ) ) @@ -86,7 +95,8 @@ module Gitlab extra.merge( import_source: :github, project_id: project_id, - importer: importer_class.name + importer: importer_class.name, + github_id: github_id ) end end diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index e5985fb94da..916b273a28f 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -14,7 +14,7 @@ module Gitlab try_import(client, project) info(project_id, message: 'stage finished') - rescue => e + rescue StandardError => e error(project_id, e) end diff --git a/app/workers/concerns/gitlab/jira_import/import_worker.rb b/app/workers/concerns/gitlab/jira_import/import_worker.rb index fdc6e64bbaa..107b6e2e9be 100644 --- a/app/workers/concerns/gitlab/jira_import/import_worker.rb +++ b/app/workers/concerns/gitlab/jira_import/import_worker.rb @@ -7,6 +7,8 @@ module Gitlab included do include ApplicationWorker + + sidekiq_options retry: 3 include ProjectImportOptions include Gitlab::JiraImport::QueueOptions end diff --git a/app/workers/concerns/limited_capacity/job_tracker.rb b/app/workers/concerns/limited_capacity/job_tracker.rb index 96b6e1a2024..47b13cd5bf6 100644 --- a/app/workers/concerns/limited_capacity/job_tracker.rb +++ b/app/workers/concerns/limited_capacity/job_tracker.rb @@ -3,21 +3,30 @@ module LimitedCapacity class JobTracker # rubocop:disable Scalability/IdempotentWorker include Gitlab::Utils::StrongMemoize + LUA_REGISTER_SCRIPT = <<~EOS + local set_key, element, max_elements = KEYS[1], ARGV[1], ARGV[2] + + if redis.call("scard", set_key) < tonumber(max_elements) then + redis.call("sadd", set_key, element) + return true + end + + return false + EOS + def initialize(namespace) @namespace = namespace end - def register(jid) - _added, @count = with_redis_pipeline do |redis| - register_job_keys(redis, jid) - get_job_count(redis) - end + def register(jid, max_jids) + with_redis do |redis| + redis.eval(LUA_REGISTER_SCRIPT, keys: [counter_key], argv: [jid, max_jids]) + end.present? end def remove(jid) - _removed, @count = with_redis_pipeline do |redis| + with_redis do |redis| remove_job_keys(redis, jid) - get_job_count(redis) end end @@ -25,14 +34,13 @@ module LimitedCapacity completed_jids = Gitlab::SidekiqStatus.completed_jids(running_jids) return unless completed_jids.any? - _removed, @count = with_redis_pipeline do |redis| + with_redis do |redis| remove_job_keys(redis, completed_jids) - get_job_count(redis) end end def count - @count ||= with_redis { |redis| get_job_count(redis) } + with_redis { |redis| redis.scard(counter_key) } end def running_jids @@ -49,14 +57,6 @@ module LimitedCapacity "worker:#{namespace.to_s.underscore}:running" end - def get_job_count(redis) - redis.scard(counter_key) - end - - def register_job_keys(redis, keys) - redis.sadd(counter_key, keys) - end - def remove_job_keys(redis, keys) redis.srem(counter_key, keys) end @@ -64,11 +64,5 @@ module LimitedCapacity def with_redis(&block) Gitlab::Redis::Queues.with(&block) # rubocop: disable CodeReuse/ActiveRecord end - - def with_redis_pipeline(&block) - with_redis do |redis| - redis.pipelined(&block) - end - end end end diff --git a/app/workers/concerns/limited_capacity/worker.rb b/app/workers/concerns/limited_capacity/worker.rb index 9dd8d942146..b4cdfda680f 100644 --- a/app/workers/concerns/limited_capacity/worker.rb +++ b/app/workers/concerns/limited_capacity/worker.rb @@ -55,26 +55,14 @@ module LimitedCapacity def perform_with_capacity(*args) worker = self.new worker.remove_failed_jobs - worker.report_prometheus_metrics(*args) - required_jobs_count = worker.required_jobs_count(*args) - arguments = Array.new(required_jobs_count) { args } + arguments = Array.new(worker.max_running_jobs) { args } self.bulk_perform_async(arguments) # rubocop:disable Scalability/BulkPerformWithContext end end def perform(*args) - return unless has_capacity? - - job_tracker.register(jid) - report_running_jobs_metrics - perform_work(*args) - rescue => exception - raise - ensure - job_tracker.remove(jid) - report_prometheus_metrics(*args) - re_enqueue(*args) unless exception + perform_registered(*args) if job_tracker.register(jid, max_running_jobs) end def perform_work(*args) @@ -89,43 +77,32 @@ module LimitedCapacity raise NotImplementedError end - def has_capacity? - remaining_capacity > 0 - end - - def remaining_capacity - [ - max_running_jobs - running_jobs_count - self.class.queue_size, - 0 - ].max - end - - def has_work?(*args) - remaining_work_count(*args) > 0 - end - def remove_failed_jobs job_tracker.clean_up end def report_prometheus_metrics(*args) report_running_jobs_metrics - remaining_work_gauge.set(prometheus_labels, remaining_work_count(*args)) - max_running_jobs_gauge.set(prometheus_labels, max_running_jobs) + set_metric(:remaining_work_gauge, remaining_work_count(*args)) + set_metric(:max_running_jobs_gauge, max_running_jobs) end - def report_running_jobs_metrics - running_jobs_gauge.set(prometheus_labels, running_jobs_count) - end + private - def required_jobs_count(*args) - [ - remaining_work_count(*args), - remaining_capacity - ].min + def perform_registered(*args) + report_running_jobs_metrics + perform_work(*args) + rescue StandardError => exception + raise + ensure + job_tracker.remove(jid) + report_prometheus_metrics(*args) + re_enqueue(*args) unless exception end - private + def report_running_jobs_metrics + set_metric(:running_jobs_gauge, running_jobs_count) + end def running_jobs_count job_tracker.count @@ -138,32 +115,21 @@ module LimitedCapacity end def re_enqueue(*args) - return unless has_capacity? - return unless has_work?(*args) + return unless remaining_work_count(*args) > 0 self.class.perform_async(*args) end - def running_jobs_gauge - strong_memoize(:running_jobs_gauge) do - Gitlab::Metrics.gauge(:limited_capacity_worker_running_jobs, 'Number of running jobs') - end - end - - def max_running_jobs_gauge - strong_memoize(:max_running_jobs_gauge) do - Gitlab::Metrics.gauge(:limited_capacity_worker_max_running_jobs, 'Maximum number of running jobs') + def set_metric(name, value) + metrics = strong_memoize(:metrics) do + { + running_jobs_gauge: Gitlab::Metrics.gauge(:limited_capacity_worker_running_jobs, 'Number of running jobs'), + max_running_jobs_gauge: Gitlab::Metrics.gauge(:limited_capacity_worker_max_running_jobs, 'Maximum number of running jobs'), + remaining_work_gauge: Gitlab::Metrics.gauge(:limited_capacity_worker_remaining_work_count, 'Number of jobs waiting to be enqueued') + } end - end - - def remaining_work_gauge - strong_memoize(:remaining_work_gauge) do - Gitlab::Metrics.gauge(:limited_capacity_worker_remaining_work_count, 'Number of jobs waiting to be enqueued') - end - end - def prometheus_labels - { worker: self.class.name } + metrics[name].set({ worker: self.class.name }, value) end end end diff --git a/app/workers/concerns/reactive_cacheable_worker.rb b/app/workers/concerns/reactive_cacheable_worker.rb index 9e882c8ac7a..78fcf8087c2 100644 --- a/app/workers/concerns/reactive_cacheable_worker.rb +++ b/app/workers/concerns/reactive_cacheable_worker.rb @@ -6,6 +6,8 @@ module ReactiveCacheableWorker included do include ApplicationWorker + sidekiq_options retry: 3 + feature_category_not_owned! loggable_arguments 0 diff --git a/app/workers/concerns/waitable_worker.rb b/app/workers/concerns/waitable_worker.rb index c3abcdafcf2..e62bd8d9885 100644 --- a/app/workers/concerns/waitable_worker.rb +++ b/app/workers/concerns/waitable_worker.rb @@ -33,7 +33,7 @@ module WaitableWorker args_list.each do |args| new.perform(*args) - rescue + rescue StandardError failed << args end diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index 6f99fd089ac..6dee9402691 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -36,13 +36,13 @@ module WorkerAttributes def feature_category(value, *extras) raise "Invalid category. Use `feature_category_not_owned!` to mark a worker as not owned" if value == :not_owned - class_attributes[:feature_category] = value + set_class_attribute(:feature_category, value) end # Special case: mark this work as not associated with a feature category # this should be used for cross-cutting concerns, such as mailer workers. def feature_category_not_owned! - class_attributes[:feature_category] = :not_owned + set_class_attribute(:feature_category, :not_owned) end def get_feature_category @@ -64,7 +64,7 @@ module WorkerAttributes def urgency(urgency) raise "Invalid urgency: #{urgency}" unless VALID_URGENCIES.include?(urgency) - class_attributes[:urgency] = urgency + set_class_attribute(:urgency, urgency) end def get_urgency @@ -75,8 +75,8 @@ module WorkerAttributes raise ArgumentError, "Invalid data consistency: #{data_consistency}" unless VALID_DATA_CONSISTENCIES.include?(data_consistency) raise ArgumentError, 'Data consistency is already set' if class_attributes[:data_consistency] - class_attributes[:data_consistency_feature_flag] = feature_flag if feature_flag - class_attributes[:data_consistency] = data_consistency + set_class_attribute(:data_consistency_feature_flag, feature_flag) if feature_flag + set_class_attribute(:data_consistency, data_consistency) validate_worker_attributes! end @@ -105,7 +105,7 @@ module WorkerAttributes # doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies for # details def worker_has_external_dependencies! - class_attributes[:external_dependencies] = true + set_class_attribute(:external_dependencies, true) end # Returns a truthy value if the worker has external dependencies. @@ -118,7 +118,7 @@ module WorkerAttributes def worker_resource_boundary(boundary) raise "Invalid boundary" unless VALID_RESOURCE_BOUNDARIES.include? boundary - class_attributes[:resource_boundary] = boundary + set_class_attribute(:resource_boundary, boundary) end def get_worker_resource_boundary @@ -126,7 +126,7 @@ module WorkerAttributes end def idempotent! - class_attributes[:idempotent] = true + set_class_attribute(:idempotent, true) validate_worker_attributes! end @@ -136,7 +136,7 @@ module WorkerAttributes end def weight(value) - class_attributes[:weight] = value + set_class_attribute(:weight, value) end def get_weight @@ -146,7 +146,7 @@ module WorkerAttributes end def tags(*values) - class_attributes[:tags] = values + set_class_attribute(:tags, values) end def get_tags @@ -154,8 +154,8 @@ module WorkerAttributes end def deduplicate(strategy, options = {}) - class_attributes[:deduplication_strategy] = strategy - class_attributes[:deduplication_options] = options + set_class_attribute(:deduplication_strategy, strategy) + set_class_attribute(:deduplication_options, options) end def get_deduplicate_strategy @@ -168,7 +168,7 @@ module WorkerAttributes end def big_payload! - class_attributes[:big_payload] = true + set_class_attribute(:big_payload, true) end def big_payload? diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb index 53220a7afed..40cc233307a 100644 --- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb +++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb @@ -3,11 +3,14 @@ module ContainerExpirationPolicies class CleanupContainerRepositoryWorker include ApplicationWorker + + sidekiq_options retry: 3 include LimitedCapacity::Worker include Gitlab::Utils::StrongMemoize queue_namespace :container_repository feature_category :container_registry + tags :exclude_from_kubernetes urgency :low worker_resource_boundary :unknown idempotent! @@ -28,7 +31,7 @@ module ContainerExpirationPolicies log_extra_metadata_on_done(:container_repository_id, container_repository.id) log_extra_metadata_on_done(:project_id, project.id) - unless allowed_to_run?(container_repository) + unless allowed_to_run? container_repository.cleanup_unscheduled! log_extra_metadata_on_done(:cleanup_status, :skipped) return @@ -39,9 +42,13 @@ module ContainerExpirationPolicies log_on_done(result) end + def max_running_jobs + return 0 unless throttling_enabled? + + ::Gitlab::CurrentSettings.container_registry_expiration_policies_worker_capacity + end + def remaining_work_count - cleanup_scheduled_count = ContainerRepository.cleanup_scheduled.count - cleanup_unfinished_count = ContainerRepository.cleanup_unfinished.count total_count = cleanup_scheduled_count + cleanup_unfinished_count log_info( @@ -53,50 +60,95 @@ module ContainerExpirationPolicies total_count end - def max_running_jobs - return 0 unless throttling_enabled? + private - ::Gitlab::CurrentSettings.current_application_settings.container_registry_expiration_policies_worker_capacity - end + def container_repository + strong_memoize(:container_repository) do + ContainerRepository.transaction do + # rubocop: disable CodeReuse/ActiveRecord + # We need a lock to prevent two workers from picking up the same row + container_repository = if loopless_enabled? + next_container_repository + else + ContainerRepository.waiting_for_cleanup + .order(:expiration_policy_cleanup_status, :expiration_policy_started_at) + .limit(1) + .lock('FOR UPDATE SKIP LOCKED') + .first + end - private + # rubocop: enable CodeReuse/ActiveRecord + container_repository&.tap(&:cleanup_ongoing!) + end + end + end - def allowed_to_run?(container_repository) - return false unless policy&.enabled && policy&.next_run_at + def next_container_repository + # rubocop: disable CodeReuse/ActiveRecord + next_one_requiring = ContainerRepository.requiring_cleanup + .order(:expiration_policy_cleanup_status, :expiration_policy_started_at) + .limit(1) + .lock('FOR UPDATE SKIP LOCKED') + .first + return next_one_requiring if next_one_requiring + + ContainerRepository.with_unfinished_cleanup + .order(:expiration_policy_started_at) + .limit(1) + .lock('FOR UPDATE SKIP LOCKED') + .first + # rubocop: enable CodeReuse/ActiveRecord + end - Time.zone.now + max_cleanup_execution_time.seconds < policy.next_run_at + def cleanup_scheduled_count + strong_memoize(:cleanup_scheduled_count) do + if loopless_enabled? + limit = max_running_jobs + 1 + ContainerExpirationPolicy.with_container_repositories + .runnable_schedules + .limit(limit) + .count + else + ContainerRepository.cleanup_scheduled.count + end + end end - def throttling_enabled? - Feature.enabled?(:container_registry_expiration_policies_throttling) + def cleanup_unfinished_count + strong_memoize(:cleanup_unfinished_count) do + if loopless_enabled? + limit = max_running_jobs + 1 + ContainerRepository.with_unfinished_cleanup + .limit(limit) + .count + else + ContainerRepository.cleanup_unfinished.count + end + end end - def max_cleanup_execution_time - ::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout + def allowed_to_run? + return false unless policy&.enabled && policy&.next_run_at + + now = Time.zone.now + + if loopless_enabled? + policy.next_run_at < now || (now + max_cleanup_execution_time.seconds < policy.next_run_at) + else + now + max_cleanup_execution_time.seconds < policy.next_run_at + end end - def policy - project.container_expiration_policy + def throttling_enabled? + Feature.enabled?(:container_registry_expiration_policies_throttling) end - def project - container_repository.project + def loopless_enabled? + Feature.enabled?(:container_registry_expiration_policies_loopless) end - def container_repository - strong_memoize(:container_repository) do - ContainerRepository.transaction do - # rubocop: disable CodeReuse/ActiveRecord - # We need a lock to prevent two workers from picking up the same row - container_repository = ContainerRepository.waiting_for_cleanup - .order(:expiration_policy_cleanup_status, :expiration_policy_started_at) - .limit(1) - .lock('FOR UPDATE SKIP LOCKED') - .first - # rubocop: enable CodeReuse/ActiveRecord - container_repository&.tap(&:cleanup_ongoing!) - end - end + def max_cleanup_execution_time + ::Gitlab::CurrentSettings.container_registry_delete_tags_service_timeout end def log_info(extra_structure) @@ -104,6 +156,11 @@ module ContainerExpirationPolicies end def log_on_done(result) + if result.error? + log_extra_metadata_on_done(:cleanup_status, :error) + log_extra_metadata_on_done(:cleanup_error_message, result.message) + end + LOG_ON_DONE_FIELDS.each do |field| value = result.payload[field] @@ -120,5 +177,13 @@ module ContainerExpirationPolicies log_extra_metadata_on_done(:cleanup_tags_service_truncated, !!truncated) log_extra_metadata_on_done(:running_jobs_count, running_jobs_count) end + + def policy + project.container_expiration_policy + end + + def project + container_repository.project + end end end diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb index 5ca89179099..dec13485d13 100644 --- a/app/workers/container_expiration_policy_worker.rb +++ b/app/workers/container_expiration_policy_worker.rb @@ -2,6 +2,8 @@ class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue include ExclusiveLeaseGuard diff --git a/app/workers/create_commit_signature_worker.rb b/app/workers/create_commit_signature_worker.rb index f81baf20d19..0ba2cc41e99 100644 --- a/app/workers/create_commit_signature_worker.rb +++ b/app/workers/create_commit_signature_worker.rb @@ -3,6 +3,8 @@ class CreateCommitSignatureWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management weight 2 idempotent! @@ -36,7 +38,7 @@ class CreateCommitSignatureWorker # This calculates and caches the signature in the database commits.each do |commit| commit&.signature - rescue => e + rescue StandardError => e Gitlab::AppLogger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}") end end diff --git a/app/workers/create_note_diff_file_worker.rb b/app/workers/create_note_diff_file_worker.rb index 06790cc89d9..0af203fc3bd 100644 --- a/app/workers/create_note_diff_file_worker.rb +++ b/app/workers/create_note_diff_file_worker.rb @@ -3,6 +3,8 @@ class CreateNoteDiffFileWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review def perform(diff_note_id) diff --git a/app/workers/create_pipeline_worker.rb b/app/workers/create_pipeline_worker.rb index 68fe44d01ce..a9072e1661f 100644 --- a/app/workers/create_pipeline_worker.rb +++ b/app/workers/create_pipeline_worker.rb @@ -2,6 +2,8 @@ class CreatePipelineWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_creation diff --git a/app/workers/database/batched_background_migration_worker.rb b/app/workers/database/batched_background_migration_worker.rb index de274d58ad7..5a326a351e8 100644 --- a/app/workers/database/batched_background_migration_worker.rb +++ b/app/workers/database/batched_background_migration_worker.rb @@ -3,9 +3,12 @@ module Database class BatchedBackgroundMigrationWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :database + tags :exclude_from_kubernetes idempotent! LEASE_TIMEOUT_MULTIPLIER = 3 @@ -13,7 +16,7 @@ module Database INTERVAL_VARIANCE = 5.seconds.freeze def perform - return unless Feature.enabled?(:execute_batched_migrations_on_schedule, type: :ops) && active_migration + return unless Feature.enabled?(:execute_batched_migrations_on_schedule, type: :ops, default_enabled: :yaml) && active_migration with_exclusive_lease(active_migration.interval) do # Now that we have the exclusive lease, reload migration in case another process has changed it. @@ -38,7 +41,7 @@ module Database end def with_exclusive_lease(interval) - timeout = max(interval * LEASE_TIMEOUT_MULTIPLIER, MINIMUM_LEASE_TIMEOUT) + timeout = [interval * LEASE_TIMEOUT_MULTIPLIER, MINIMUM_LEASE_TIMEOUT].max lease = Gitlab::ExclusiveLease.new(lease_key, timeout: timeout) yield if lease.try_obtain @@ -46,10 +49,6 @@ module Database lease&.cancel end - def max(left, right) - left >= right ? left : right - end - def lease_key self.class.name.demodulize.underscore end diff --git a/app/workers/delete_container_repository_worker.rb b/app/workers/delete_container_repository_worker.rb index dbfc273a5ce..f39f8bf44a4 100644 --- a/app/workers/delete_container_repository_worker.rb +++ b/app/workers/delete_container_repository_worker.rb @@ -2,6 +2,8 @@ class DeleteContainerRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExclusiveLeaseGuard queue_namespace :container_repository diff --git a/app/workers/delete_diff_files_worker.rb b/app/workers/delete_diff_files_worker.rb index 289df8873ec..46dac5d8d39 100644 --- a/app/workers/delete_diff_files_worker.rb +++ b/app/workers/delete_diff_files_worker.rb @@ -3,6 +3,8 @@ class DeleteDiffFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/delete_merged_branches_worker.rb b/app/workers/delete_merged_branches_worker.rb index 8d7026e2d1e..c7e1a4da965 100644 --- a/app/workers/delete_merged_branches_worker.rb +++ b/app/workers/delete_merged_branches_worker.rb @@ -3,6 +3,8 @@ class DeleteMergedBranchesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management def perform(project_id, user_id) diff --git a/app/workers/delete_stored_files_worker.rb b/app/workers/delete_stored_files_worker.rb index 9cf5631b7d8..75113b4787c 100644 --- a/app/workers/delete_stored_files_worker.rb +++ b/app/workers/delete_stored_files_worker.rb @@ -3,6 +3,8 @@ class DeleteStoredFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category_not_owned! loggable_arguments 0 diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb index ed2e00f1241..f1b9f859ce6 100644 --- a/app/workers/delete_user_worker.rb +++ b/app/workers/delete_user_worker.rb @@ -3,6 +3,8 @@ class DeleteUserWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :authentication_and_authorization loggable_arguments 2 diff --git a/app/workers/deployments/drop_older_deployments_worker.rb b/app/workers/deployments/drop_older_deployments_worker.rb index d6cd92c1da4..6ca819e7942 100644 --- a/app/workers/deployments/drop_older_deployments_worker.rb +++ b/app/workers/deployments/drop_older_deployments_worker.rb @@ -4,8 +4,11 @@ module Deployments class DropOlderDeploymentsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :deployment feature_category :continuous_delivery + tags :exclude_from_kubernetes def perform(deployment_id) Deployments::OlderDeploymentsDropService.new(deployment_id).execute diff --git a/app/workers/deployments/execute_hooks_worker.rb b/app/workers/deployments/execute_hooks_worker.rb index 6be05232321..3046aa28e20 100644 --- a/app/workers/deployments/execute_hooks_worker.rb +++ b/app/workers/deployments/execute_hooks_worker.rb @@ -1,16 +1,19 @@ # frozen_string_literal: true module Deployments + # TODO: remove in https://gitlab.com/gitlab-org/gitlab/-/issues/329360 class ExecuteHooksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :deployment feature_category :continuous_delivery worker_resource_boundary :cpu def perform(deployment_id) if (deploy = Deployment.find_by_id(deployment_id)) - deploy.execute_hooks + deploy.execute_hooks(Time.current) end end end diff --git a/app/workers/deployments/finished_worker.rb b/app/workers/deployments/finished_worker.rb index 62c886010a3..3de06c381cd 100644 --- a/app/workers/deployments/finished_worker.rb +++ b/app/workers/deployments/finished_worker.rb @@ -6,6 +6,8 @@ module Deployments class FinishedWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :deployment feature_category :continuous_delivery worker_resource_boundary :cpu @@ -13,7 +15,7 @@ module Deployments def perform(deployment_id) if (deploy = Deployment.find_by_id(deployment_id)) LinkMergeRequestsService.new(deploy).execute - deploy.execute_hooks + deploy.execute_hooks(Time.current) end end end diff --git a/app/workers/deployments/forward_deployment_worker.rb b/app/workers/deployments/forward_deployment_worker.rb index dd01fcbbafe..946945051ba 100644 --- a/app/workers/deployments/forward_deployment_worker.rb +++ b/app/workers/deployments/forward_deployment_worker.rb @@ -6,6 +6,8 @@ module Deployments class ForwardDeploymentWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :deployment feature_category :continuous_delivery diff --git a/app/workers/deployments/hooks_worker.rb b/app/workers/deployments/hooks_worker.rb new file mode 100644 index 00000000000..beac44881fb --- /dev/null +++ b/app/workers/deployments/hooks_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Deployments + class HooksWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :deployment + feature_category :continuous_delivery + + def perform(params = {}) + params = params.with_indifferent_access + + if (deploy = Deployment.find_by_id(params[:deployment_id])) + deploy.execute_hooks(params[:status_changed_at].to_time) + end + end + end +end diff --git a/app/workers/deployments/link_merge_request_worker.rb b/app/workers/deployments/link_merge_request_worker.rb index 4723691a0bb..70947b3f731 100644 --- a/app/workers/deployments/link_merge_request_worker.rb +++ b/app/workers/deployments/link_merge_request_worker.rb @@ -4,6 +4,8 @@ module Deployments class LinkMergeRequestWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :deployment idempotent! feature_category :continuous_delivery diff --git a/app/workers/deployments/success_worker.rb b/app/workers/deployments/success_worker.rb index b72b107985b..eab331433e8 100644 --- a/app/workers/deployments/success_worker.rb +++ b/app/workers/deployments/success_worker.rb @@ -6,6 +6,8 @@ module Deployments class SuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :deployment feature_category :continuous_delivery worker_resource_boundary :cpu diff --git a/app/workers/deployments/update_environment_worker.rb b/app/workers/deployments/update_environment_worker.rb index 2381f9926bc..5c71a13064e 100644 --- a/app/workers/deployments/update_environment_worker.rb +++ b/app/workers/deployments/update_environment_worker.rb @@ -4,6 +4,8 @@ module Deployments class UpdateEnvironmentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :deployment idempotent! feature_category :continuous_delivery diff --git a/app/workers/design_management/copy_design_collection_worker.rb b/app/workers/design_management/copy_design_collection_worker.rb index 0a6e23fe9da..28b511c7c27 100644 --- a/app/workers/design_management/copy_design_collection_worker.rb +++ b/app/workers/design_management/copy_design_collection_worker.rb @@ -4,7 +4,10 @@ module DesignManagement class CopyDesignCollectionWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :design_management + tags :exclude_from_kubernetes idempotent! urgency :low diff --git a/app/workers/design_management/new_version_worker.rb b/app/workers/design_management/new_version_worker.rb index 4fbf2067be4..eee96858c34 100644 --- a/app/workers/design_management/new_version_worker.rb +++ b/app/workers/design_management/new_version_worker.rb @@ -4,6 +4,8 @@ module DesignManagement class NewVersionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :design_management # Declare this worker as memory bound due to # `GenerateImageVersionsService` resizing designs diff --git a/app/workers/destroy_pages_deployments_worker.rb b/app/workers/destroy_pages_deployments_worker.rb index 32b539325c9..edd446628aa 100644 --- a/app/workers/destroy_pages_deployments_worker.rb +++ b/app/workers/destroy_pages_deployments_worker.rb @@ -8,6 +8,7 @@ class DestroyPagesDeploymentsWorker loggable_arguments 0, 1 sidekiq_options retry: 3 feature_category :pages + tags :exclude_from_kubernetes def perform(project_id, last_deployment_id = nil) project = Project.find_by_id(project_id) diff --git a/app/workers/disallow_two_factor_for_group_worker.rb b/app/workers/disallow_two_factor_for_group_worker.rb index b3cc7a44672..3a48e3ab5da 100644 --- a/app/workers/disallow_two_factor_for_group_worker.rb +++ b/app/workers/disallow_two_factor_for_group_worker.rb @@ -2,9 +2,12 @@ class DisallowTwoFactorForGroupWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExceptionBacktrace feature_category :subgroups + tags :exclude_from_kubernetes idempotent! def perform(group_id) diff --git a/app/workers/disallow_two_factor_for_subgroups_worker.rb b/app/workers/disallow_two_factor_for_subgroups_worker.rb index 1ca227030e2..f5b31e0bcf0 100644 --- a/app/workers/disallow_two_factor_for_subgroups_worker.rb +++ b/app/workers/disallow_two_factor_for_subgroups_worker.rb @@ -2,11 +2,14 @@ class DisallowTwoFactorForSubgroupsWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExceptionBacktrace INTERVAL = 2.seconds.to_i feature_category :subgroups + tags :exclude_from_kubernetes idempotent! def perform(group_id) diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 9ceab9bb878..37ed1001c9d 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -3,25 +3,86 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :issue_tracking urgency :high weight 2 + attr_accessor :raw + def perform(raw) - return unless Gitlab::IncomingEmail.enabled? + return unless should_perform? - begin - Gitlab::Email::Receiver.new(raw).execute - rescue => e - handle_failure(raw, e) - end + @raw = raw + execute_receiver + end + + def should_perform? + Gitlab::IncomingEmail.enabled? end private - def handle_failure(raw, error) - Gitlab::AppLogger.warn("Email can not be processed: #{error}\n\n#{raw}") + def execute_receiver + receiver.execute + log_success + rescue StandardError => e + log_error(e) + handle_failure(e) + end + + def receiver + @receiver ||= Gitlab::Email::Receiver.new(raw) + end + + def logger + Sidekiq.logger + end + + def log_success + logger.info(build_message('Successfully processed message', receiver.mail_metadata)) + end + + def log_error(error) + payload = + case error + # Unparsable e-mails don't have metadata we can use + when Gitlab::Email::EmailUnparsableError, Gitlab::Email::EmptyEmailError + {} + else + mail_metadata + end + + # We don't need the backtrace and more details if the e-mail couldn't be processed + if error.is_a?(Gitlab::Email::ProcessingError) + payload['exception.class'] = error.class.name + else + Gitlab::ExceptionLogFormatter.format!(error, payload) + Gitlab::ErrorTracking.track_exception(error) + end + + logger.error(build_message('Error processing message', payload)) + end + + def build_message(message, params = {}) + { + class: self.class.name, + Labkit::Correlation::CorrelationId::LOG_KEY => Labkit::Correlation::CorrelationId.current_id, + message: message + }.merge(params) + end + + def mail_metadata + receiver.mail_metadata + rescue StandardError => e + # We should never get here as long as we check EmailUnparsableError, but + # let's be defensive in case we did something wrong. + Gitlab::ErrorTracking.track_exception(e) + {} + end + def handle_failure(error) return unless raw.present? can_retry = false diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index 978b65802dd..9c4418c5f31 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -3,6 +3,8 @@ class EmailsOnPushWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + attr_reader :email, :skip_premailer feature_category :source_code_management @@ -56,7 +58,7 @@ class EmailsOnPushWorker # rubocop:disable Scalability/IdempotentWorker end end - EmailsOnPushService.valid_recipients(recipients).each do |recipient| + Integrations::EmailsOnPush.valid_recipients(recipients).each do |recipient| send_email( recipient, project_id, diff --git a/app/workers/environments/auto_stop_cron_worker.rb b/app/workers/environments/auto_stop_cron_worker.rb index ada52d3402d..64028839df1 100644 --- a/app/workers/environments/auto_stop_cron_worker.rb +++ b/app/workers/environments/auto_stop_cron_worker.rb @@ -3,6 +3,8 @@ module Environments class AutoStopCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :continuous_delivery diff --git a/app/workers/environments/canary_ingress/update_worker.rb b/app/workers/environments/canary_ingress/update_worker.rb index 53cc38e9eec..ecdfc6f0581 100644 --- a/app/workers/environments/canary_ingress/update_worker.rb +++ b/app/workers/environments/canary_ingress/update_worker.rb @@ -9,6 +9,7 @@ module Environments idempotent! worker_has_external_dependencies! feature_category :continuous_delivery + tags :exclude_from_kubernetes def perform(environment_id, params) Environment.find_by_id(environment_id).try do |environment| diff --git a/app/workers/error_tracking_issue_link_worker.rb b/app/workers/error_tracking_issue_link_worker.rb index 4ad80d57f6b..6c5a96822a6 100644 --- a/app/workers/error_tracking_issue_link_worker.rb +++ b/app/workers/error_tracking_issue_link_worker.rb @@ -7,6 +7,8 @@ # until the prior link is deleted. class ErrorTrackingIssueLinkWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExclusiveLeaseGuard include Gitlab::Utils::StrongMemoize diff --git a/app/workers/experiments/record_conversion_event_worker.rb b/app/workers/experiments/record_conversion_event_worker.rb index e38ce7b3d01..9fc76a2173b 100644 --- a/app/workers/experiments/record_conversion_event_worker.rb +++ b/app/workers/experiments/record_conversion_event_worker.rb @@ -4,7 +4,10 @@ module Experiments class RecordConversionEventWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :users + tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb index 50fdd046491..a9fa94ef301 100644 --- a/app/workers/expire_build_artifacts_worker.rb +++ b/app/workers/expire_build_artifacts_worker.rb @@ -2,6 +2,8 @@ class ExpireBuildArtifactsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index e6cd60a3e47..3e6e81867bd 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -3,6 +3,8 @@ class ExpireBuildInstanceArtifactsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :continuous_integration # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index 48bb1160ae8..074c35997f6 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -2,6 +2,8 @@ class ExpireJobCacheWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_cache diff --git a/app/workers/expire_pipeline_cache_worker.rb b/app/workers/expire_pipeline_cache_worker.rb index cbea46cdccd..3c48c4ba3cd 100644 --- a/app/workers/expire_pipeline_cache_worker.rb +++ b/app/workers/expire_pipeline_cache_worker.rb @@ -2,6 +2,8 @@ class ExpirePipelineCacheWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_cache diff --git a/app/workers/export_csv_worker.rb b/app/workers/export_csv_worker.rb index f2da381a34a..a2ad0cb92fd 100644 --- a/app/workers/export_csv_worker.rb +++ b/app/workers/export_csv_worker.rb @@ -3,6 +3,8 @@ class ExportCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :issue_tracking worker_resource_boundary :cpu loggable_arguments 2 diff --git a/app/workers/flush_counter_increments_worker.rb b/app/workers/flush_counter_increments_worker.rb index b7e3c0c134d..44d30b4ba3d 100644 --- a/app/workers/flush_counter_increments_worker.rb +++ b/app/workers/flush_counter_increments_worker.rb @@ -8,7 +8,10 @@ class FlushCounterIncrementsWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category_not_owned! + tags :exclude_from_kubernetes urgency :low deduplicate :until_executing, including_scheduled: true diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb deleted file mode 100644 index a2aab23db7b..00000000000 --- a/app/workers/git_garbage_collect_worker.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -# According to our docs, we can only remove workers on major releases -# https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#removing-workers. -# -# We need to still maintain this until 14.0 but with the current functionality. -# -# In https://gitlab.com/gitlab-org/gitlab/-/issues/299290 we track that removal. -class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - sidekiq_options retry: false - feature_category :gitaly - loggable_arguments 1, 2, 3 - - def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil) - ::Projects::GitGarbageCollectWorker.new.perform(project_id, task, lease_key, lease_uuid) - end -end diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index af406b32415..f25296f0461 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -8,6 +8,8 @@ module Gitlab # stage. class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ::Gitlab::Import::AdvanceStage sidekiq_options dead: false diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb index 79ef917bbc5..a8b79cf9b3a 100644 --- a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -5,6 +5,8 @@ module Gitlab class ImportPullRequestMergedByWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter + tags :exclude_from_kubernetes + def representation_class Gitlab::GithubImport::Representation::PullRequest end diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb index b8516fb8670..5ee88d5d32b 100644 --- a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -5,6 +5,8 @@ module Gitlab class ImportPullRequestReviewWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter + tags :exclude_from_kubernetes + def representation_class Gitlab::GithubImport::Representation::PullRequestReview end diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb index 0ddd893d0d1..1c769921ab3 100644 --- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb +++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport class RefreshImportJidWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue # The interval to schedule new instances of this job at. diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index 058e1a0853d..f5980cc248e 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index 202bb335ca1..7ca23ecad20 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class ImportBaseDataWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index 486057804b4..d66698277b0 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class ImportIssuesAndDiffNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb index de2a7f9fc29..2a66a08d534 100644 --- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class ImportLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index e1da26a9d48..873e389fca6 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb index 3e15c346659..5743648680d 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb @@ -5,9 +5,13 @@ module Gitlab module Stage class ImportPullRequestsMergedByWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods + tags :exclude_from_kubernetes + # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb index 790e8b0eccf..532d550f190 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb @@ -5,9 +5,13 @@ module Gitlab module Stage class ImportPullRequestsReviewsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods + tags :exclude_from_kubernetes + # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index bf2defa6326..5755aea21ce 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index 3338f7e58c0..e113563ce8b 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/import/stuck_import_job.rb b/app/workers/gitlab/import/stuck_import_job.rb index 16be7a77ab1..ac789ce1188 100644 --- a/app/workers/gitlab/import/stuck_import_job.rb +++ b/app/workers/gitlab/import/stuck_import_job.rb @@ -9,6 +9,8 @@ module Gitlab included do include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker updates several import states inline and does not schedule # other jobs. So no context needed diff --git a/app/workers/gitlab/jira_import/advance_stage_worker.rb b/app/workers/gitlab/jira_import/advance_stage_worker.rb index c3a64669c60..6387054d448 100644 --- a/app/workers/gitlab/jira_import/advance_stage_worker.rb +++ b/app/workers/gitlab/jira_import/advance_stage_worker.rb @@ -4,6 +4,8 @@ module Gitlab module JiraImport class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include QueueOptions include ::Gitlab::Import::AdvanceStage diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index d1ceda4fd6a..98bde2218c2 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -4,6 +4,8 @@ module Gitlab module JiraImport class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include NotifyUponDeath include Gitlab::JiraImport::QueueOptions include Gitlab::Import::DatabaseHelpers @@ -13,7 +15,7 @@ module Gitlab def perform(project_id, jira_issue_id, issue_attributes, waiter_key) issue_id = create_issue(issue_attributes, project_id) JiraImport.cache_issue_mapping(issue_id, jira_issue_id, project_id) - rescue => ex + rescue StandardError => ex # Todo: Record jira issue id(or better jira issue key), # so that we can report the list of failed to import issues to the user # see https://gitlab.com/gitlab-org/gitlab/-/issues/211653 diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb index bfc02224ee4..e327ced8c65 100644 --- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -5,6 +5,8 @@ module Gitlab module Stage class StartImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ProjectStartImport include ProjectImportOptions include Gitlab::JiraImport::QueueOptions diff --git a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb index 1b1d7b35dd5..867a12fbac2 100644 --- a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb +++ b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb @@ -3,6 +3,8 @@ module Gitlab module PhabricatorImport class ImportTasksWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ProjectImportOptions # This marks the project as failed after too many tries def importer_class diff --git a/app/workers/gitlab_performance_bar_stats_worker.rb b/app/workers/gitlab_performance_bar_stats_worker.rb index 558df0ab7b3..4f7fdcf96f0 100644 --- a/app/workers/gitlab_performance_bar_stats_worker.rb +++ b/app/workers/gitlab_performance_bar_stats_worker.rb @@ -3,6 +3,8 @@ class GitlabPerformanceBarStatsWorker include ApplicationWorker + sidekiq_options retry: 3 + LEASE_KEY = 'gitlab:performance_bar_stats' LEASE_TIMEOUT = 600 WORKER_DELAY = 120 @@ -10,6 +12,7 @@ class GitlabPerformanceBarStatsWorker STATS_KEY_EXPIRE = 30.minutes.to_i feature_category :metrics + tags :exclude_from_kubernetes idempotent! def perform(lease_uuid) diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index b8e1e3d8fc4..de1e9af7bae 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -2,6 +2,8 @@ class GitlabShellWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include Gitlab::ShellAdapter feature_category :source_code_management diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb index 901785f462b..2c140c89e26 100644 --- a/app/workers/group_destroy_worker.rb +++ b/app/workers/group_destroy_worker.rb @@ -2,10 +2,12 @@ class GroupDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExceptionBacktrace feature_category :subgroups - tags :requires_disk_io + tags :requires_disk_io, :exclude_from_kubernetes def perform(group_id, user_id) begin diff --git a/app/workers/hashed_storage/migrator_worker.rb b/app/workers/hashed_storage/migrator_worker.rb index 5cbdfcb0602..735d8a2447a 100644 --- a/app/workers/hashed_storage/migrator_worker.rb +++ b/app/workers/hashed_storage/migrator_worker.rb @@ -4,8 +4,11 @@ module HashedStorage class MigratorWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :hashed_storage feature_category :source_code_management + tags :exclude_from_gitlab_com # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index 03e53058dbb..0659c8a6a46 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -4,8 +4,11 @@ module HashedStorage class ProjectMigrateWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :hashed_storage loggable_arguments 1 + tags :exclude_from_gitlab_com attr_reader :project_id diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index d4a5e474323..a5ee8b35176 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -4,8 +4,11 @@ module HashedStorage class ProjectRollbackWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :hashed_storage loggable_arguments 1 + tags :exclude_from_gitlab_com attr_reader :project_id diff --git a/app/workers/hashed_storage/rollbacker_worker.rb b/app/workers/hashed_storage/rollbacker_worker.rb index a220d3b2226..447bdfa6220 100644 --- a/app/workers/hashed_storage/rollbacker_worker.rb +++ b/app/workers/hashed_storage/rollbacker_worker.rb @@ -4,8 +4,11 @@ module HashedStorage class RollbackerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :hashed_storage feature_category :source_code_management + tags :exclude_from_gitlab_com # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/import_export_project_cleanup_worker.rb b/app/workers/import_export_project_cleanup_worker.rb index dd345434d08..6e112a47932 100644 --- a/app/workers/import_export_project_cleanup_worker.rb +++ b/app/workers/import_export_project_cleanup_worker.rb @@ -2,6 +2,8 @@ class ImportExportProjectCleanupWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb index 521e5b8fbc2..46b59dc398f 100644 --- a/app/workers/import_issues_csv_worker.rb +++ b/app/workers/import_issues_csv_worker.rb @@ -3,6 +3,8 @@ class ImportIssuesCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! feature_category :issue_tracking worker_resource_boundary :cpu diff --git a/app/workers/incident_management/add_severity_system_note_worker.rb b/app/workers/incident_management/add_severity_system_note_worker.rb index 9f132531562..62ed902e488 100644 --- a/app/workers/incident_management/add_severity_system_note_worker.rb +++ b/app/workers/incident_management/add_severity_system_note_worker.rb @@ -4,8 +4,11 @@ module IncidentManagement class AddSeveritySystemNoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :incident_management feature_category :incident_management + tags :exclude_from_kubernetes def perform(incident_id, user_id) return if incident_id.blank? || user_id.blank? diff --git a/app/workers/incident_management/pager_duty/process_incident_worker.rb b/app/workers/incident_management/pager_duty/process_incident_worker.rb index 3f378b012a1..413a297a024 100644 --- a/app/workers/incident_management/pager_duty/process_incident_worker.rb +++ b/app/workers/incident_management/pager_duty/process_incident_worker.rb @@ -5,6 +5,8 @@ module IncidentManagement class ProcessIncidentWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :incident_management feature_category :incident_management diff --git a/app/workers/incident_management/process_alert_worker.rb b/app/workers/incident_management/process_alert_worker.rb index 59464b81d1b..3b90e296ad4 100644 --- a/app/workers/incident_management/process_alert_worker.rb +++ b/app/workers/incident_management/process_alert_worker.rb @@ -4,12 +4,20 @@ module IncidentManagement class ProcessAlertWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :incident_management feature_category :incident_management # `project_id` and `alert_payload` are deprecated and can be removed # starting from 14.0 release # https://gitlab.com/gitlab-org/gitlab/-/issues/224500 + # + # This worker is not scheduled anymore since + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60285 + # and will be removed completely via + # https://gitlab.com/gitlab-org/gitlab/-/issues/224500 + # in 14.0. def perform(_project_id = nil, _alert_payload = nil, alert_id = nil) return unless alert_id diff --git a/app/workers/incident_management/process_alert_worker_v2.rb b/app/workers/incident_management/process_alert_worker_v2.rb new file mode 100644 index 00000000000..04bf6970578 --- /dev/null +++ b/app/workers/incident_management/process_alert_worker_v2.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module IncidentManagement + class ProcessAlertWorkerV2 # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :incident_management + feature_category :incident_management + + idempotent! + + def perform(alert_id) + return unless alert_id + + alert = find_alert(alert_id) + return unless alert + + result = create_issue_for(alert) + return if result.success? + + log_warning(alert, result) + end + + private + + def find_alert(alert_id) + AlertManagement::Alert.find_by_id(alert_id) + end + + def create_issue_for(alert) + AlertManagement::CreateAlertIssueService + .new(alert, User.alert_bot) + .execute + end + + def log_warning(alert, result) + issue_id = result.payload[:issue]&.id + + Gitlab::AppLogger.warn( + message: 'Cannot process an Incident', + issue_id: issue_id, + alert_id: alert.id, + errors: result.message + ) + end + end +end diff --git a/app/workers/incident_management/process_prometheus_alert_worker.rb b/app/workers/incident_management/process_prometheus_alert_worker.rb index 4b778f6a621..7b5c6fd9001 100644 --- a/app/workers/incident_management/process_prometheus_alert_worker.rb +++ b/app/workers/incident_management/process_prometheus_alert_worker.rb @@ -4,6 +4,8 @@ module IncidentManagement class ProcessPrometheusAlertWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :incident_management feature_category :incident_management worker_resource_boundary :cpu diff --git a/app/workers/invalid_gpg_signature_update_worker.rb b/app/workers/invalid_gpg_signature_update_worker.rb index 1fd959c8763..662817b5a92 100644 --- a/app/workers/invalid_gpg_signature_update_worker.rb +++ b/app/workers/invalid_gpg_signature_update_worker.rb @@ -3,6 +3,8 @@ class InvalidGpgSignatureUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management weight 2 diff --git a/app/workers/irker_worker.rb b/app/workers/irker_worker.rb index c5bdb3e0970..4378da186a7 100644 --- a/app/workers/irker_worker.rb +++ b/app/workers/irker_worker.rb @@ -6,6 +6,8 @@ require 'socket' class IrkerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations def perform(project_id, channels, colors, push_data, settings) diff --git a/app/workers/issuable/label_links_destroy_worker.rb b/app/workers/issuable/label_links_destroy_worker.rb new file mode 100644 index 00000000000..f663c410fba --- /dev/null +++ b/app/workers/issuable/label_links_destroy_worker.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Issuable + class LabelLinksDestroyWorker + include ApplicationWorker + + idempotent! + feature_category :issue_tracking + + def perform(target_id, target_type) + ::Issuable::DestroyLabelLinksService.new(target_id, target_type).execute + end + end +end diff --git a/app/workers/issuable_export_csv_worker.rb b/app/workers/issuable_export_csv_worker.rb index eb96a78497c..41facab6bb9 100644 --- a/app/workers/issuable_export_csv_worker.rb +++ b/app/workers/issuable_export_csv_worker.rb @@ -3,6 +3,8 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :issue_tracking worker_resource_boundary :cpu loggable_arguments 2 @@ -47,4 +49,4 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker end end -IssuableExportCsvWorker.prepend_if_ee('::EE::IssuableExportCsvWorker') +IssuableExportCsvWorker.prepend_mod_with('IssuableExportCsvWorker') diff --git a/app/workers/issuables/clear_groups_issue_counter_worker.rb b/app/workers/issuables/clear_groups_issue_counter_worker.rb new file mode 100644 index 00000000000..a8d6fd2f870 --- /dev/null +++ b/app/workers/issuables/clear_groups_issue_counter_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Issuables + class ClearGroupsIssueCounterWorker + include ApplicationWorker + + idempotent! + urgency :low + feature_category :issue_tracking + + def perform(group_ids = []) + return if group_ids.empty? + + groups_with_ancestors = Gitlab::ObjectHierarchy + .new(Group.by_id(group_ids)) + .base_and_ancestors + + clear_cached_count(groups_with_ancestors) + end + + private + + def clear_cached_count(groups) + groups.each do |group| + Groups::OpenIssuesCountService.new(group).clear_all_cache_keys + end + end + end +end diff --git a/app/workers/issue_due_scheduler_worker.rb b/app/workers/issue_due_scheduler_worker.rb index d735295d046..9077b42d645 100644 --- a/app/workers/issue_due_scheduler_worker.rb +++ b/app/workers/issue_due_scheduler_worker.rb @@ -2,6 +2,8 @@ class IssueDueSchedulerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :issue_tracking diff --git a/app/workers/issue_placement_worker.rb b/app/workers/issue_placement_worker.rb index 5b547ab0c8d..dba791c3f05 100644 --- a/app/workers/issue_placement_worker.rb +++ b/app/workers/issue_placement_worker.rb @@ -3,7 +3,10 @@ class IssuePlacementWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! + deduplicate :until_executed, including_scheduled: true feature_category :issue_tracking urgency :high worker_resource_boundary :cpu @@ -17,6 +20,10 @@ class IssuePlacementWorker issue = find_issue(issue_id, project_id) return unless issue + # Temporary disable moving null elements because of performance problems + # For more information check https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4321 + return if issue.blocked_for_repositioning? + # Move the oldest 100 unpositioned items to the end. # This is to deal with out-of-order execution of the worker, # while preserving creation order. @@ -30,7 +37,7 @@ class IssuePlacementWorker leftover = to_place.pop if to_place.count > QUERY_LIMIT Issue.move_nulls_to_end(to_place) - Issues::BaseService.new(nil).rebalance_if_needed(to_place.max_by(&:relative_position)) + Issues::BaseService.new(project: nil).rebalance_if_needed(to_place.max_by(&:relative_position)) IssuePlacementWorker.perform_async(nil, leftover.project_id) if leftover.present? rescue RelativePositioning::NoSpaceLeft => e Gitlab::ErrorTracking.log_exception(e, issue_id: issue_id, project_id: project_id) diff --git a/app/workers/issue_rebalancing_worker.rb b/app/workers/issue_rebalancing_worker.rb index a9ad66198f3..9eac451f107 100644 --- a/app/workers/issue_rebalancing_worker.rb +++ b/app/workers/issue_rebalancing_worker.rb @@ -3,14 +3,22 @@ class IssueRebalancingWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! urgency :low feature_category :issue_tracking + tags :exclude_from_kubernetes def perform(ignore = nil, project_id = nil) return if project_id.nil? project = Project.find(project_id) + + # Temporary disable reabalancing for performance reasons + # For more information check https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4321 + return if project.root_namespace&.issue_repositioning_disabled? + # All issues are equivalent as far as we are concerned issue = project.issues.take # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/jira_connect/sync_branch_worker.rb b/app/workers/jira_connect/sync_branch_worker.rb index 1af51c4bb74..b8211286d1c 100644 --- a/app/workers/jira_connect/sync_branch_worker.rb +++ b/app/workers/jira_connect/sync_branch_worker.rb @@ -4,6 +4,8 @@ module JiraConnect class SyncBranchWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :jira_connect feature_category :integrations loggable_arguments 1, 2 diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb index 9cb5d5d247d..11a3b598035 100644 --- a/app/workers/jira_connect/sync_builds_worker.rb +++ b/app/workers/jira_connect/sync_builds_worker.rb @@ -4,11 +4,14 @@ module JiraConnect class SyncBuildsWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! worker_has_external_dependencies! queue_namespace :jira_connect feature_category :integrations + tags :exclude_from_kubernetes def perform(pipeline_id, sequence_id) pipeline = Ci::Pipeline.find_by_id(pipeline_id) diff --git a/app/workers/jira_connect/sync_deployments_worker.rb b/app/workers/jira_connect/sync_deployments_worker.rb index 7272d35f4cb..9f75b1161f0 100644 --- a/app/workers/jira_connect/sync_deployments_worker.rb +++ b/app/workers/jira_connect/sync_deployments_worker.rb @@ -4,11 +4,14 @@ module JiraConnect class SyncDeploymentsWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! worker_has_external_dependencies! queue_namespace :jira_connect feature_category :integrations + tags :exclude_from_kubernetes def perform(deployment_id, sequence_id) deployment = Deployment.find_by_id(deployment_id) diff --git a/app/workers/jira_connect/sync_feature_flags_worker.rb b/app/workers/jira_connect/sync_feature_flags_worker.rb index 496b9f1626d..0d8d3d3142e 100644 --- a/app/workers/jira_connect/sync_feature_flags_worker.rb +++ b/app/workers/jira_connect/sync_feature_flags_worker.rb @@ -4,11 +4,14 @@ module JiraConnect class SyncFeatureFlagsWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! worker_has_external_dependencies! queue_namespace :jira_connect feature_category :integrations + tags :exclude_from_kubernetes def perform(feature_flag_id, sequence_id) feature_flag = ::Operations::FeatureFlag.find_by_id(feature_flag_id) diff --git a/app/workers/jira_connect/sync_merge_request_worker.rb b/app/workers/jira_connect/sync_merge_request_worker.rb index 543d8e002fe..6b3a6ae84ad 100644 --- a/app/workers/jira_connect/sync_merge_request_worker.rb +++ b/app/workers/jira_connect/sync_merge_request_worker.rb @@ -4,6 +4,8 @@ module JiraConnect class SyncMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :jira_connect feature_category :integrations idempotent! diff --git a/app/workers/jira_connect/sync_project_worker.rb b/app/workers/jira_connect/sync_project_worker.rb index 4d52705f207..dfff0c4b3b6 100644 --- a/app/workers/jira_connect/sync_project_worker.rb +++ b/app/workers/jira_connect/sync_project_worker.rb @@ -4,8 +4,11 @@ module JiraConnect class SyncProjectWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :jira_connect feature_category :integrations + tags :exclude_from_kubernetes idempotent! worker_has_external_dependencies! diff --git a/app/workers/mail_scheduler/issue_due_worker.rb b/app/workers/mail_scheduler/issue_due_worker.rb index 309d3e13477..44f8f1e446c 100644 --- a/app/workers/mail_scheduler/issue_due_worker.rb +++ b/app/workers/mail_scheduler/issue_due_worker.rb @@ -3,6 +3,8 @@ module MailScheduler class IssueDueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include MailSchedulerQueue feature_category :issue_tracking diff --git a/app/workers/mail_scheduler/notification_service_worker.rb b/app/workers/mail_scheduler/notification_service_worker.rb index 309f23c8708..8645cc93511 100644 --- a/app/workers/mail_scheduler/notification_service_worker.rb +++ b/app/workers/mail_scheduler/notification_service_worker.rb @@ -5,6 +5,8 @@ require 'active_job/arguments' module MailScheduler class NotificationServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include MailSchedulerQueue feature_category :issue_tracking diff --git a/app/workers/member_invitation_reminder_emails_worker.rb b/app/workers/member_invitation_reminder_emails_worker.rb index 971d6abaa51..bfee8ab1fab 100644 --- a/app/workers/member_invitation_reminder_emails_worker.rb +++ b/app/workers/member_invitation_reminder_emails_worker.rb @@ -2,9 +2,12 @@ class MemberInvitationReminderEmailsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :subgroups + tags :exclude_from_kubernetes urgency :low def perform diff --git a/app/workers/members_destroyer/unassign_issuables_worker.rb b/app/workers/members_destroyer/unassign_issuables_worker.rb index 2c17120bf48..0ee5654eaba 100644 --- a/app/workers/members_destroyer/unassign_issuables_worker.rb +++ b/app/workers/members_destroyer/unassign_issuables_worker.rb @@ -4,6 +4,8 @@ module MembersDestroyer class UnassignIssuablesWorker include ApplicationWorker + sidekiq_options retry: 3 + ENTITY_TYPES = %w(Group Project).freeze queue_namespace :unassign_issuables diff --git a/app/workers/merge_request_cleanup_refs_worker.rb b/app/workers/merge_request_cleanup_refs_worker.rb index fbd62ac0a91..162c6dc2a88 100644 --- a/app/workers/merge_request_cleanup_refs_worker.rb +++ b/app/workers/merge_request_cleanup_refs_worker.rb @@ -3,7 +3,10 @@ class MergeRequestCleanupRefsWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review + tags :exclude_from_kubernetes idempotent! def perform(merge_request_id) diff --git a/app/workers/merge_request_mergeability_check_worker.rb b/app/workers/merge_request_mergeability_check_worker.rb index 70d5f49d70e..13961de1f59 100644 --- a/app/workers/merge_request_mergeability_check_worker.rb +++ b/app/workers/merge_request_mergeability_check_worker.rb @@ -3,6 +3,8 @@ class MergeRequestMergeabilityCheckWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review idempotent! diff --git a/app/workers/merge_requests/assignees_change_worker.rb b/app/workers/merge_requests/assignees_change_worker.rb index 9865563e357..fe39f20151f 100644 --- a/app/workers/merge_requests/assignees_change_worker.rb +++ b/app/workers/merge_requests/assignees_change_worker.rb @@ -3,6 +3,8 @@ class MergeRequests::AssigneesChangeWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management urgency :high deduplicate :until_executed @@ -19,7 +21,7 @@ class MergeRequests::AssigneesChangeWorker return if users.blank? ::MergeRequests::HandleAssigneesChangeService - .new(merge_request.target_project, current_user) + .new(project: merge_request.target_project, current_user: current_user) .execute(merge_request, users, execute_hooks: true) rescue ActiveRecord::RecordNotFound end diff --git a/app/workers/merge_requests/create_pipeline_worker.rb b/app/workers/merge_requests/create_pipeline_worker.rb index 244ba1af300..a79a92a5419 100644 --- a/app/workers/merge_requests/create_pipeline_worker.rb +++ b/app/workers/merge_requests/create_pipeline_worker.rb @@ -3,6 +3,8 @@ module MergeRequests class CreatePipelineWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_creation @@ -21,7 +23,7 @@ module MergeRequests merge_request = MergeRequest.find_by_id(merge_request_id) return unless merge_request - MergeRequests::CreatePipelineService.new(project, user).execute(merge_request) + MergeRequests::CreatePipelineService.new(project: project, current_user: user).execute(merge_request) merge_request.update_head_pipeline end end diff --git a/app/workers/merge_requests/delete_source_branch_worker.rb b/app/workers/merge_requests/delete_source_branch_worker.rb index eb83d10af33..1ce3a99b298 100644 --- a/app/workers/merge_requests/delete_source_branch_worker.rb +++ b/app/workers/merge_requests/delete_source_branch_worker.rb @@ -3,6 +3,8 @@ class MergeRequests::DeleteSourceBranchWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management urgency :high idempotent! @@ -17,7 +19,7 @@ class MergeRequests::DeleteSourceBranchWorker ::Branches::DeleteService.new(merge_request.source_project, user) .execute(merge_request.source_branch) - ::MergeRequests::RetargetChainService.new(merge_request.source_project, user) + ::MergeRequests::RetargetChainService.new(project: merge_request.source_project, current_user: user) .execute(merge_request) rescue ActiveRecord::RecordNotFound end diff --git a/app/workers/merge_requests/handle_assignees_change_worker.rb b/app/workers/merge_requests/handle_assignees_change_worker.rb index e79d8293bae..4c0500cd520 100644 --- a/app/workers/merge_requests/handle_assignees_change_worker.rb +++ b/app/workers/merge_requests/handle_assignees_change_worker.rb @@ -3,6 +3,8 @@ class MergeRequests::HandleAssigneesChangeWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review urgency :high deduplicate :until_executed @@ -15,7 +17,7 @@ class MergeRequests::HandleAssigneesChangeWorker old_assignees = User.id_in(old_assignee_ids) ::MergeRequests::HandleAssigneesChangeService - .new(merge_request.target_project, user) + .new(project: merge_request.target_project, current_user: user) .execute(merge_request, old_assignees, options) rescue ActiveRecord::RecordNotFound end diff --git a/app/workers/merge_requests/resolve_todos_worker.rb b/app/workers/merge_requests/resolve_todos_worker.rb index 2a5f742f809..8bb88091efe 100644 --- a/app/workers/merge_requests/resolve_todos_worker.rb +++ b/app/workers/merge_requests/resolve_todos_worker.rb @@ -3,6 +3,8 @@ class MergeRequests::ResolveTodosWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review urgency :high deduplicate :until_executed diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index 270bd831f96..df5a7a904fc 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -3,17 +3,27 @@ class MergeWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management urgency :high weight 5 loggable_arguments 2 + idempotent! + + deduplicate :until_executed, including_scheduled: true def perform(merge_request_id, current_user_id, params) params = params.with_indifferent_access - current_user = User.find(current_user_id) - merge_request = MergeRequest.find(merge_request_id) - MergeRequests::MergeService.new(merge_request.target_project, current_user, params) + begin + current_user = User.find(current_user_id) + merge_request = MergeRequest.find(merge_request_id) + rescue ActiveRecord::RecordNotFound + return + end + + MergeRequests::MergeService.new(project: merge_request.target_project, current_user: current_user, params: params) .execute(merge_request) end end diff --git a/app/workers/metrics/dashboard/prune_old_annotations_worker.rb b/app/workers/metrics/dashboard/prune_old_annotations_worker.rb index 2a9ce3bb8e6..5e8067a4438 100644 --- a/app/workers/metrics/dashboard/prune_old_annotations_worker.rb +++ b/app/workers/metrics/dashboard/prune_old_annotations_worker.rb @@ -5,6 +5,8 @@ module Metrics class PruneOldAnnotationsWorker include ApplicationWorker + sidekiq_options retry: 3 + DELETE_LIMIT = 10_000 DEFAULT_CUT_OFF_PERIOD = 2.weeks diff --git a/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb b/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb index cbdd69c6e8c..6f2ff8cca13 100644 --- a/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb +++ b/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb @@ -4,6 +4,8 @@ module Metrics module Dashboard class ScheduleAnnotationsPruneWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/metrics/dashboard/sync_dashboards_worker.rb b/app/workers/metrics/dashboard/sync_dashboards_worker.rb index 7a124a33f9e..0fdc7825f47 100644 --- a/app/workers/metrics/dashboard/sync_dashboards_worker.rb +++ b/app/workers/metrics/dashboard/sync_dashboards_worker.rb @@ -5,7 +5,10 @@ module Metrics class SyncDashboardsWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :metrics + tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/migrate_external_diffs_worker.rb b/app/workers/migrate_external_diffs_worker.rb index 3ef399bd9fc..a73a9be4f0c 100644 --- a/app/workers/migrate_external_diffs_worker.rb +++ b/app/workers/migrate_external_diffs_worker.rb @@ -3,6 +3,8 @@ class MigrateExternalDiffsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review def perform(merge_request_diff_id) diff --git a/app/workers/namespaceless_project_destroy_worker.rb b/app/workers/namespaceless_project_destroy_worker.rb index 1c8054d8fbd..91cad6f2a5c 100644 --- a/app/workers/namespaceless_project_destroy_worker.rb +++ b/app/workers/namespaceless_project_destroy_worker.rb @@ -8,6 +8,8 @@ # namespace. For those use ProjectDestroyWorker instead. class NamespacelessProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExceptionBacktrace feature_category :authentication_and_authorization diff --git a/app/workers/namespaces/in_product_marketing_emails_worker.rb b/app/workers/namespaces/in_product_marketing_emails_worker.rb index 3070afed3d6..7985325d1ad 100644 --- a/app/workers/namespaces/in_product_marketing_emails_worker.rb +++ b/app/workers/namespaces/in_product_marketing_emails_worker.rb @@ -3,9 +3,12 @@ module Namespaces class InProductMarketingEmailsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :subgroups + tags :exclude_from_kubernetes urgency :low def perform @@ -32,4 +35,4 @@ module Namespaces end end -Namespaces::InProductMarketingEmailsWorker.prepend_if_ee('EE::Namespaces::InProductMarketingEmailsWorker') +Namespaces::InProductMarketingEmailsWorker.prepend_mod_with('Namespaces::InProductMarketingEmailsWorker') diff --git a/app/workers/namespaces/onboarding_issue_created_worker.rb b/app/workers/namespaces/onboarding_issue_created_worker.rb index e5e2c80e821..7b8b1a43078 100644 --- a/app/workers/namespaces/onboarding_issue_created_worker.rb +++ b/app/workers/namespaces/onboarding_issue_created_worker.rb @@ -4,7 +4,10 @@ module Namespaces class OnboardingIssueCreatedWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :issue_tracking + tags :exclude_from_kubernetes urgency :low deduplicate :until_executing diff --git a/app/workers/namespaces/onboarding_pipeline_created_worker.rb b/app/workers/namespaces/onboarding_pipeline_created_worker.rb index e1de6d0046b..128d7b6aa06 100644 --- a/app/workers/namespaces/onboarding_pipeline_created_worker.rb +++ b/app/workers/namespaces/onboarding_pipeline_created_worker.rb @@ -4,7 +4,10 @@ module Namespaces class OnboardingPipelineCreatedWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :subgroups + tags :exclude_from_kubernetes urgency :low deduplicate :until_executing diff --git a/app/workers/namespaces/onboarding_progress_worker.rb b/app/workers/namespaces/onboarding_progress_worker.rb index 9cb5a23cf31..d4db55a9207 100644 --- a/app/workers/namespaces/onboarding_progress_worker.rb +++ b/app/workers/namespaces/onboarding_progress_worker.rb @@ -4,7 +4,10 @@ module Namespaces class OnboardingProgressWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :product_analytics + tags :exclude_from_kubernetes urgency :low deduplicate :until_executed diff --git a/app/workers/namespaces/onboarding_user_added_worker.rb b/app/workers/namespaces/onboarding_user_added_worker.rb index 02608268d6f..8c85dfafa12 100644 --- a/app/workers/namespaces/onboarding_user_added_worker.rb +++ b/app/workers/namespaces/onboarding_user_added_worker.rb @@ -4,7 +4,10 @@ module Namespaces class OnboardingUserAddedWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :users + tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/namespaces/prune_aggregation_schedules_worker.rb b/app/workers/namespaces/prune_aggregation_schedules_worker.rb index b94c8b7b4ba..0ea27c532ae 100644 --- a/app/workers/namespaces/prune_aggregation_schedules_worker.rb +++ b/app/workers/namespaces/prune_aggregation_schedules_worker.rb @@ -3,6 +3,8 @@ module Namespaces class PruneAggregationSchedulesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :source_code_management diff --git a/app/workers/namespaces/root_statistics_worker.rb b/app/workers/namespaces/root_statistics_worker.rb index 5d4b510ceb7..92bf2e22020 100644 --- a/app/workers/namespaces/root_statistics_worker.rb +++ b/app/workers/namespaces/root_statistics_worker.rb @@ -4,6 +4,8 @@ module Namespaces class RootStatisticsWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :update_namespace_statistics feature_category :source_code_management idempotent! diff --git a/app/workers/namespaces/schedule_aggregation_worker.rb b/app/workers/namespaces/schedule_aggregation_worker.rb index cbf5ed44572..cee273688e9 100644 --- a/app/workers/namespaces/schedule_aggregation_worker.rb +++ b/app/workers/namespaces/schedule_aggregation_worker.rb @@ -4,6 +4,8 @@ module Namespaces class ScheduleAggregationWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :update_namespace_statistics feature_category :source_code_management idempotent! diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb index c08f4b4cd75..a579b828354 100644 --- a/app/workers/new_issue_worker.rb +++ b/app/workers/new_issue_worker.rb @@ -2,6 +2,8 @@ class NewIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include NewIssuable feature_category :issue_tracking @@ -18,7 +20,7 @@ class NewIssueWorker # rubocop:disable Scalability/IdempotentWorker issuable.create_cross_references!(user) Issues::AfterCreateService - .new(issuable.project, user) + .new(project: issuable.project, current_user: user) .execute(issuable) end diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb index 2d28561488b..574c73ad3b5 100644 --- a/app/workers/new_merge_request_worker.rb +++ b/app/workers/new_merge_request_worker.rb @@ -2,6 +2,8 @@ class NewMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include NewIssuable feature_category :code_review @@ -13,7 +15,7 @@ class NewMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker return unless objects_found?(merge_request_id, user_id) MergeRequests::AfterCreateService - .new(issuable.target_project, user) + .new(project: issuable.target_project, current_user: user) .execute(issuable) end diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index 2bb2d0db55c..566bb9a9057 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -3,6 +3,8 @@ class NewNoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :issue_tracking urgency :high worker_resource_boundary :cpu diff --git a/app/workers/object_pool/create_worker.rb b/app/workers/object_pool/create_worker.rb index cf87ad95077..586b81fcd30 100644 --- a/app/workers/object_pool/create_worker.rb +++ b/app/workers/object_pool/create_worker.rb @@ -3,6 +3,8 @@ module ObjectPool class CreateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ObjectPoolQueue include ExclusiveLeaseGuard @@ -28,7 +30,7 @@ module ObjectPool pool.create_object_pool pool.mark_ready - rescue => e + rescue StandardError => e pool.mark_failed raise e end diff --git a/app/workers/object_pool/destroy_worker.rb b/app/workers/object_pool/destroy_worker.rb index d42cee59d03..297780b20bd 100644 --- a/app/workers/object_pool/destroy_worker.rb +++ b/app/workers/object_pool/destroy_worker.rb @@ -3,6 +3,8 @@ module ObjectPool class DestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ObjectPoolQueue def perform(pool_repository_id) diff --git a/app/workers/object_pool/join_worker.rb b/app/workers/object_pool/join_worker.rb index 8103c04b507..282a8f54695 100644 --- a/app/workers/object_pool/join_worker.rb +++ b/app/workers/object_pool/join_worker.rb @@ -3,6 +3,8 @@ module ObjectPool class JoinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ObjectPoolQueue worker_resource_boundary :cpu diff --git a/app/workers/object_pool/schedule_join_worker.rb b/app/workers/object_pool/schedule_join_worker.rb index c00bb2967f2..44208208d04 100644 --- a/app/workers/object_pool/schedule_join_worker.rb +++ b/app/workers/object_pool/schedule_join_worker.rb @@ -3,6 +3,8 @@ module ObjectPool class ScheduleJoinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ObjectPoolQueue def perform(pool_id) diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb index 666bacb0188..7323ab50370 100644 --- a/app/workers/object_storage/migrate_uploads_worker.rb +++ b/app/workers/object_storage/migrate_uploads_worker.rb @@ -4,6 +4,8 @@ module ObjectStorage class MigrateUploadsWorker include ApplicationWorker + + sidekiq_options retry: 3 include ObjectStorageQueue feature_category_not_owned! @@ -48,7 +50,7 @@ module ObjectStorage Gitlab::AppLogger.info header(success, failures) Gitlab::AppLogger.warn failures(failures) - raise MigrationFailures.new(failures.map(&:error)) if failures.any? + raise MigrationFailures, failures.map(&:error) if failures.any? end def header(success, failures) @@ -132,7 +134,7 @@ module ObjectStorage def process_uploader(uploader) MigrationResult.new(uploader.upload).tap do |result| uploader.migrate!(@to_store) - rescue => e + rescue StandardError => e result.error = e end end diff --git a/app/workers/packages/composer/cache_cleanup_worker.rb b/app/workers/packages/composer/cache_cleanup_worker.rb index 638e50e18c4..1d47ef87962 100644 --- a/app/workers/packages/composer/cache_cleanup_worker.rb +++ b/app/workers/packages/composer/cache_cleanup_worker.rb @@ -4,9 +4,12 @@ module Packages module Composer class CacheCleanupWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :package_registry + tags :exclude_from_kubernetes idempotent! @@ -22,7 +25,7 @@ module Packages rescue ActiveRecord::RecordNotFound # ignore. likely due to object already being deleted. end - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) end end diff --git a/app/workers/packages/composer/cache_update_worker.rb b/app/workers/packages/composer/cache_update_worker.rb index 664fb23284f..d87abf2e256 100644 --- a/app/workers/packages/composer/cache_update_worker.rb +++ b/app/workers/packages/composer/cache_update_worker.rb @@ -5,7 +5,10 @@ module Packages class CacheUpdateWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :package_registry + tags :exclude_from_kubernetes idempotent! @@ -15,7 +18,7 @@ module Packages return unless project Gitlab::Composer::Cache.new(project: project, name: package_name, last_page_sha: last_page_sha).execute - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e, project_id: project_id) end end diff --git a/app/workers/packages/debian/process_changes_worker.rb b/app/workers/packages/debian/process_changes_worker.rb new file mode 100644 index 00000000000..edc366a7597 --- /dev/null +++ b/app/workers/packages/debian/process_changes_worker.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Packages + module Debian + class ProcessChangesWorker + include ApplicationWorker + include Gitlab::Utils::StrongMemoize + + deduplicate :until_executed + idempotent! + + queue_namespace :package_repositories + feature_category :package_registry + tags :exclude_from_kubernetes + + def perform(package_file_id, user_id) + @package_file_id = package_file_id + @user_id = user_id + + return unless package_file && user + + ::Packages::Debian::ProcessChangesService.new(package_file, user).execute + rescue ArgumentError, + Packages::Debian::ExtractChangesMetadataService::ExtractionError, + Packages::Debian::ExtractDebMetadataService::CommandFailedError, + Packages::Debian::ExtractMetadataService::ExtractionError, + Packages::Debian::ParseDebian822Service::InvalidDebian822Error, + ActiveRecord::RecordNotFound => e + Gitlab::ErrorTracking.log_exception(e, package_file_id: @package_file_id, user_id: @user_id) + package_file.destroy! + end + + private + + attr_reader :package_file_id, :user_id + + def package_file + strong_memoize(:package_file) do + ::Packages::PackageFile.find_by_id(package_file_id) + end + end + + def user + strong_memoize(:user) do + ::User.find_by_id(user_id) + end + end + end + end +end diff --git a/app/workers/packages/go/sync_packages_worker.rb b/app/workers/packages/go/sync_packages_worker.rb index e41f27f2252..c5f631c47db 100644 --- a/app/workers/packages/go/sync_packages_worker.rb +++ b/app/workers/packages/go/sync_packages_worker.rb @@ -4,10 +4,13 @@ module Packages module Go class SyncPackagesWorker include ApplicationWorker + + sidekiq_options retry: 3 include Gitlab::Golang queue_namespace :package_repositories feature_category :package_registry + tags :exclude_from_kubernetes deduplicate :until_executing idempotent! diff --git a/app/workers/packages/maven/metadata/sync_worker.rb b/app/workers/packages/maven/metadata/sync_worker.rb index eb7abf4cdd0..c53117a08c5 100644 --- a/app/workers/packages/maven/metadata/sync_worker.rb +++ b/app/workers/packages/maven/metadata/sync_worker.rb @@ -5,10 +5,13 @@ module Packages module Metadata class SyncWorker include ApplicationWorker + + sidekiq_options retry: 3 include Gitlab::Utils::StrongMemoize queue_namespace :package_repositories feature_category :package_registry + tags :exclude_from_kubernetes deduplicate :until_executing idempotent! @@ -30,10 +33,10 @@ module Packages if result.success? log_extra_metadata_on_done(:message, result.message) else - raise SyncError.new(result.message) + raise SyncError, result.message end - raise SyncError.new(result.message) unless result.success? + raise SyncError, result.message unless result.success? end private diff --git a/app/workers/packages/nuget/extraction_worker.rb b/app/workers/packages/nuget/extraction_worker.rb index 820304a9f3b..4128b229ebe 100644 --- a/app/workers/packages/nuget/extraction_worker.rb +++ b/app/workers/packages/nuget/extraction_worker.rb @@ -5,6 +5,8 @@ module Packages class ExtractionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :package_repositories feature_category :package_registry @@ -15,10 +17,9 @@ module Packages ::Packages::Nuget::UpdatePackageFromMetadataService.new(package_file).execute - rescue ::Packages::Nuget::MetadataExtractionService::ExtractionError, - ::Packages::Nuget::UpdatePackageFromMetadataService::InvalidMetadataError => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e, project_id: package_file.project_id) - package_file.package.destroy! + package_file.package.update_column(:status, :error) end end end diff --git a/app/workers/packages/rubygems/extraction_worker.rb b/app/workers/packages/rubygems/extraction_worker.rb index 1e5cd0b54ce..fc32654a2c1 100644 --- a/app/workers/packages/rubygems/extraction_worker.rb +++ b/app/workers/packages/rubygems/extraction_worker.rb @@ -5,12 +5,13 @@ module Packages class ExtractionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + queue_namespace :package_repositories feature_category :package_registry + tags :exclude_from_kubernetes deduplicate :until_executing - idempotent! - def perform(package_file_id) package_file = ::Packages::PackageFile.find_by_id(package_file_id) @@ -18,9 +19,9 @@ module Packages ::Packages::Rubygems::ProcessGemService.new(package_file).execute - rescue ::Packages::Rubygems::ProcessGemService::ExtractionError => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e, project_id: package_file.project_id) - package_file.package.destroy! + package_file.package.update_column(:status, :error) end end end diff --git a/app/workers/pages_domain_removal_cron_worker.rb b/app/workers/pages_domain_removal_cron_worker.rb index cb24441d2f7..cc720676214 100644 --- a/app/workers/pages_domain_removal_cron_worker.rb +++ b/app/workers/pages_domain_removal_cron_worker.rb @@ -2,6 +2,8 @@ class PagesDomainRemovalCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :pages @@ -10,7 +12,7 @@ class PagesDomainRemovalCronWorker # rubocop:disable Scalability/IdempotentWorke def perform PagesDomain.for_removal.with_logging_info.find_each do |domain| with_context(project: domain.project) { domain.destroy! } - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e) end end diff --git a/app/workers/pages_domain_ssl_renewal_cron_worker.rb b/app/workers/pages_domain_ssl_renewal_cron_worker.rb index fe6d516d3cf..c99eed8a8df 100644 --- a/app/workers/pages_domain_ssl_renewal_cron_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_cron_worker.rb @@ -2,6 +2,8 @@ class PagesDomainSslRenewalCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :pages diff --git a/app/workers/pages_domain_ssl_renewal_worker.rb b/app/workers/pages_domain_ssl_renewal_worker.rb index 125ba343948..2ab41aab795 100644 --- a/app/workers/pages_domain_ssl_renewal_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_worker.rb @@ -3,8 +3,10 @@ class PagesDomainSslRenewalWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :pages - tags :requires_disk_io + tags :requires_disk_io, :exclude_from_kubernetes def perform(domain_id) domain = PagesDomain.find_by_id(domain_id) diff --git a/app/workers/pages_domain_verification_cron_worker.rb b/app/workers/pages_domain_verification_cron_worker.rb index a30f0b981d8..ec63004716a 100644 --- a/app/workers/pages_domain_verification_cron_worker.rb +++ b/app/workers/pages_domain_verification_cron_worker.rb @@ -2,6 +2,8 @@ class PagesDomainVerificationCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :pages diff --git a/app/workers/pages_domain_verification_worker.rb b/app/workers/pages_domain_verification_worker.rb index ff0463481cd..b67b1b4d8ee 100644 --- a/app/workers/pages_domain_verification_worker.rb +++ b/app/workers/pages_domain_verification_worker.rb @@ -3,8 +3,10 @@ class PagesDomainVerificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :pages - tags :requires_disk_io + tags :requires_disk_io, :exclude_from_kubernetes # rubocop: disable CodeReuse/ActiveRecord def perform(domain_id) diff --git a/app/workers/pages_remove_worker.rb b/app/workers/pages_remove_worker.rb index 67ea18545a7..3e60df9027a 100644 --- a/app/workers/pages_remove_worker.rb +++ b/app/workers/pages_remove_worker.rb @@ -6,6 +6,7 @@ class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 feature_category :pages + tags :exclude_from_kubernetes loggable_arguments 0 def perform(project_id) diff --git a/app/workers/pages_transfer_worker.rb b/app/workers/pages_transfer_worker.rb index 5d395c9e38a..0d80ec28310 100644 --- a/app/workers/pages_transfer_worker.rb +++ b/app/workers/pages_transfer_worker.rb @@ -3,9 +3,12 @@ class PagesTransferWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + TransferFailedError = Class.new(StandardError) feature_category :pages + tags :exclude_from_kubernetes loggable_arguments 0, 1 def perform(method, args) diff --git a/app/workers/pages_update_configuration_worker.rb b/app/workers/pages_update_configuration_worker.rb index ca5016dc782..8bb9f76670b 100644 --- a/app/workers/pages_update_configuration_worker.rb +++ b/app/workers/pages_update_configuration_worker.rb @@ -3,8 +3,11 @@ class PagesUpdateConfigurationWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! feature_category :pages + tags :exclude_from_kubernetes def self.perform_async(*args) return unless ::Settings.pages.local_store.enabled diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index 0c561626f8c..ee394271653 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -6,7 +6,7 @@ class PagesWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 feature_category :pages loggable_arguments 0, 1 - tags :requires_disk_io + tags :requires_disk_io, :exclude_from_kubernetes def perform(action, *arg) send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/workers/partition_creation_worker.rb b/app/workers/partition_creation_worker.rb index 119ecd28003..2b21741d6c2 100644 --- a/app/workers/partition_creation_worker.rb +++ b/app/workers/partition_creation_worker.rb @@ -2,6 +2,8 @@ class PartitionCreationWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :database diff --git a/app/workers/personal_access_tokens/expired_notification_worker.rb b/app/workers/personal_access_tokens/expired_notification_worker.rb index 2ff64ec51f3..73568960d38 100644 --- a/app/workers/personal_access_tokens/expired_notification_worker.rb +++ b/app/workers/personal_access_tokens/expired_notification_worker.rb @@ -3,9 +3,12 @@ module PersonalAccessTokens class ExpiredNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :authentication_and_authorization + tags :exclude_from_kubernetes def perform(*args) notification_service = NotificationService.new diff --git a/app/workers/personal_access_tokens/expiring_worker.rb b/app/workers/personal_access_tokens/expiring_worker.rb index 7a016c85a64..aaca78e3c63 100644 --- a/app/workers/personal_access_tokens/expiring_worker.rb +++ b/app/workers/personal_access_tokens/expiring_worker.rb @@ -3,6 +3,8 @@ module PersonalAccessTokens class ExpiringWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :authentication_and_authorization diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb index b8dd4768cfb..fbb672f52e3 100644 --- a/app/workers/pipeline_hooks_worker.rb +++ b/app/workers/pipeline_hooks_worker.rb @@ -2,6 +2,8 @@ class PipelineHooksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_hooks diff --git a/app/workers/pipeline_metrics_worker.rb b/app/workers/pipeline_metrics_worker.rb index 1eb9b4ce089..fdab10d7dda 100644 --- a/app/workers/pipeline_metrics_worker.rb +++ b/app/workers/pipeline_metrics_worker.rb @@ -2,6 +2,8 @@ class PipelineMetricsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue urgency :high diff --git a/app/workers/pipeline_notification_worker.rb b/app/workers/pipeline_notification_worker.rb index f4b43106bf2..619570dcf41 100644 --- a/app/workers/pipeline_notification_worker.rb +++ b/app/workers/pipeline_notification_worker.rb @@ -2,6 +2,8 @@ class PipelineNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue urgency :high diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb index f0929b92bd0..dc14789fe73 100644 --- a/app/workers/pipeline_process_worker.rb +++ b/app/workers/pipeline_process_worker.rb @@ -2,12 +2,15 @@ class PipelineProcessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing feature_category :continuous_integration urgency :high loggable_arguments 1 + data_consistency :delayed, feature_flag: :load_balancing_for_pipeline_process_worker # rubocop: disable CodeReuse/ActiveRecord # `_build_ids` is deprecated and will be removed in 14.0 diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb index d81b978f9b0..f1248ec9e58 100644 --- a/app/workers/pipeline_schedule_worker.rb +++ b/app/workers/pipeline_schedule_worker.rb @@ -2,6 +2,8 @@ class PipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :continuous_integration diff --git a/app/workers/pipeline_update_worker.rb b/app/workers/pipeline_update_worker.rb index 7db4ab8fe0b..e8feb4f2db2 100644 --- a/app/workers/pipeline_update_worker.rb +++ b/app/workers/pipeline_update_worker.rb @@ -4,6 +4,8 @@ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/232806 class PipelineUpdateWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 313b901c08e..ce985492935 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -2,6 +2,8 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include Gitlab::Experiment::Dsl feature_category :source_code_management @@ -123,7 +125,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker def after_project_changes_hooks(project, user, refs, changes) experiment(:new_project_readme, actor: user).track_initial_writes(project) - experiment(:empty_repo_upload, project: project).track(:initial_write) if project.empty_repo? + experiment(:empty_repo_upload, project: project).track_initial_write repository_update_hook_data = Gitlab::DataBuilder::Repository.update(project, user, changes, refs) SystemHooksService.new.execute_hooks(repository_update_hook_data, :repository_update_hooks) Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes) @@ -134,4 +136,4 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker end end -PostReceive.prepend_if_ee('EE::PostReceive') +PostReceive.prepend_mod_with('PostReceive') diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index 3c7af641f16..54ffe8d3b10 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -10,6 +10,8 @@ class ProcessCommitWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management urgency :high weight 3 @@ -51,7 +53,7 @@ class ProcessCommitWorker # therefore we use IssueCollection here and skip the authorization check in # Issues::CloseService#execute. IssueCollection.new(issues).updatable_by_user(user).each do |issue| - Issues::CloseService.new(project, author) + Issues::CloseService.new(project: project, current_user: author) .close_issue(issue, closed_via: commit) end end diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index 8a9c166e5df..d2796cdb697 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -4,6 +4,8 @@ class ProjectCacheWorker include ApplicationWorker + sidekiq_options retry: 3 + LEASE_TIMEOUT = 15.minutes.to_i feature_category :source_code_management @@ -61,4 +63,4 @@ class ProjectCacheWorker end end -ProjectCacheWorker.prepend_if_ee('EE::ProjectCacheWorker') +ProjectCacheWorker.prepend_mod_with('ProjectCacheWorker') diff --git a/app/workers/project_daily_statistics_worker.rb b/app/workers/project_daily_statistics_worker.rb index 2166655115d..7d673ec00d9 100644 --- a/app/workers/project_daily_statistics_worker.rb +++ b/app/workers/project_daily_statistics_worker.rb @@ -4,6 +4,8 @@ class ProjectDailyStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management def perform(project_id) diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb index 99d51fc5c2e..be11fa65028 100644 --- a/app/workers/project_destroy_worker.rb +++ b/app/workers/project_destroy_worker.rb @@ -2,10 +2,12 @@ class ProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExceptionBacktrace feature_category :source_code_management - tags :requires_disk_io + tags :requires_disk_io, :exclude_from_kubernetes def perform(project_id, user_id, params) project = Project.find(project_id) diff --git a/app/workers/project_service_worker.rb b/app/workers/project_service_worker.rb index 84c3a3e52d0..967be3b3e81 100644 --- a/app/workers/project_service_worker.rb +++ b/app/workers/project_service_worker.rb @@ -3,16 +3,18 @@ class ProjectServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + sidekiq_options dead: false feature_category :integrations worker_has_external_dependencies! def perform(hook_id, data) data = data.with_indifferent_access - service = Service.find(hook_id) - service.execute(data) - rescue => error - service_class = service&.class&.name || "Not Found" - logger.error class: self.class.name, service_class: service_class, message: error.message + integration = Integration.find(hook_id) + integration.execute(data) + rescue StandardError => error + integration_class = integration&.class&.name || "Not Found" + logger.error class: self.class.name, service_class: integration_class, message: error.message end end diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb index 4f908529b34..0d67a8ac30e 100644 --- a/app/workers/projects/git_garbage_collect_worker.rb +++ b/app/workers/projects/git_garbage_collect_worker.rb @@ -5,6 +5,8 @@ module Projects extend ::Gitlab::Utils::Override include GitGarbageCollectMethods + tags :exclude_from_kubernetes + private override :find_resource @@ -24,7 +26,7 @@ module Projects return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary ::Gitlab::Cleanup::OrphanLfsFileReferences.new(resource, dry_run: false, logger: logger).run! - rescue => err + rescue StandardError => err Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err) end diff --git a/app/workers/projects/post_creation_worker.rb b/app/workers/projects/post_creation_worker.rb index 2ca62e582b6..1970f79729f 100644 --- a/app/workers/projects/post_creation_worker.rb +++ b/app/workers/projects/post_creation_worker.rb @@ -4,7 +4,10 @@ module Projects class PostCreationWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management + tags :exclude_from_kubernetes idempotent! def perform(project_id) diff --git a/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb b/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb index 3841ae9b922..55530bff7c1 100644 --- a/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb +++ b/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb @@ -4,6 +4,8 @@ module Projects class ScheduleBulkRepositoryShardMovesWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! feature_category :gitaly urgency :throttled diff --git a/app/workers/projects/update_repository_storage_worker.rb b/app/workers/projects/update_repository_storage_worker.rb index f4c44458446..1de1c95e043 100644 --- a/app/workers/projects/update_repository_storage_worker.rb +++ b/app/workers/projects/update_repository_storage_worker.rb @@ -5,6 +5,8 @@ module Projects extend ::Gitlab::Utils::Override include ::UpdateRepositoryStorageWorker + sidekiq_options retry: 3 + private override :find_repository_storage_move diff --git a/app/workers/prometheus/create_default_alerts_worker.rb b/app/workers/prometheus/create_default_alerts_worker.rb index 2c4fefa9ece..0dba752ced1 100644 --- a/app/workers/prometheus/create_default_alerts_worker.rb +++ b/app/workers/prometheus/create_default_alerts_worker.rb @@ -4,6 +4,8 @@ module Prometheus class CreateDefaultAlertsWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :incident_management urgency :high idempotent! diff --git a/app/workers/propagate_integration_group_worker.rb b/app/workers/propagate_integration_group_worker.rb index 01155753877..6881740461f 100644 --- a/app/workers/propagate_integration_group_worker.rb +++ b/app/workers/propagate_integration_group_worker.rb @@ -3,15 +3,18 @@ class PropagateIntegrationGroupWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations + tags :exclude_from_kubernetes idempotent! # rubocop: disable CodeReuse/ActiveRecord def perform(integration_id, min_id, max_id) - integration = Service.find_by_id(integration_id) + integration = Integration.find_by_id(integration_id) return unless integration - batch = if integration.instance? + batch = if integration.instance_level? Group.where(id: min_id..max_id).without_integration(integration) else integration.group.descendants.where(id: min_id..max_id).without_integration(integration) diff --git a/app/workers/propagate_integration_inherit_descendant_worker.rb b/app/workers/propagate_integration_inherit_descendant_worker.rb index d589619818c..9067af12de3 100644 --- a/app/workers/propagate_integration_inherit_descendant_worker.rb +++ b/app/workers/propagate_integration_inherit_descendant_worker.rb @@ -3,15 +3,18 @@ class PropagateIntegrationInheritDescendantWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations + tags :exclude_from_kubernetes idempotent! # rubocop: disable CodeReuse/ActiveRecord def perform(integration_id, min_id, max_id) - integration = Service.find_by_id(integration_id) + integration = Integration.find_by_id(integration_id) return unless integration - batch = Service.inherited_descendants_from_self_or_ancestors_from(integration).where(id: min_id..max_id) + batch = Integration.inherited_descendants_from_self_or_ancestors_from(integration).where(id: min_id..max_id) BulkUpdateIntegrationService.new(integration, batch).execute end diff --git a/app/workers/propagate_integration_inherit_worker.rb b/app/workers/propagate_integration_inherit_worker.rb index 40d67c6d3bf..e7649d6714f 100644 --- a/app/workers/propagate_integration_inherit_worker.rb +++ b/app/workers/propagate_integration_inherit_worker.rb @@ -3,15 +3,18 @@ class PropagateIntegrationInheritWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations + tags :exclude_from_kubernetes idempotent! # rubocop: disable CodeReuse/ActiveRecord def perform(integration_id, min_id, max_id) - integration = Service.find_by_id(integration_id) + integration = Integration.find_by_id(integration_id) return unless integration - batch = Service.where(id: min_id..max_id).by_type(integration.type).inherit_from_id(integration.id) + batch = Integration.where(id: min_id..max_id).by_type(integration.type).inherit_from_id(integration.id) BulkUpdateIntegrationService.new(integration, batch).execute end diff --git a/app/workers/propagate_integration_project_worker.rb b/app/workers/propagate_integration_project_worker.rb index 188d81e5fc1..90cf27c4176 100644 --- a/app/workers/propagate_integration_project_worker.rb +++ b/app/workers/propagate_integration_project_worker.rb @@ -3,16 +3,19 @@ class PropagateIntegrationProjectWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations + tags :exclude_from_kubernetes idempotent! # rubocop: disable CodeReuse/ActiveRecord def perform(integration_id, min_id, max_id) - integration = Service.find_by_id(integration_id) + integration = Integration.find_by_id(integration_id) return unless integration batch = Project.where(id: min_id..max_id).without_integration(integration) - batch = batch.in_namespace(integration.group.self_and_descendants) if integration.group_id + batch = batch.in_namespace(integration.group.self_and_descendants) if integration.group_level? return if batch.empty? diff --git a/app/workers/propagate_integration_worker.rb b/app/workers/propagate_integration_worker.rb index bb954b12a25..5e694529bc0 100644 --- a/app/workers/propagate_integration_worker.rb +++ b/app/workers/propagate_integration_worker.rb @@ -3,6 +3,8 @@ class PropagateIntegrationWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations idempotent! loggable_arguments 1 @@ -10,6 +12,6 @@ class PropagateIntegrationWorker # TODO: Keep overwrite parameter for backwards compatibility. Remove after >= 14.0 # https://gitlab.com/gitlab-org/gitlab/-/issues/255382 def perform(integration_id, overwrite = nil) - Admin::PropagateIntegrationService.propagate(Service.find(integration_id)) + Admin::PropagateIntegrationService.propagate(Integration.find(integration_id)) end end diff --git a/app/workers/propagate_service_template_worker.rb b/app/workers/propagate_service_template_worker.rb index b02525b5106..149577b15cd 100644 --- a/app/workers/propagate_service_template_worker.rb +++ b/app/workers/propagate_service_template_worker.rb @@ -4,6 +4,8 @@ class PropagateServiceTemplateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations LEASE_TIMEOUT = 4.hours.to_i @@ -12,7 +14,7 @@ class PropagateServiceTemplateWorker # rubocop:disable Scalability/IdempotentWor def perform(template_id) return unless try_obtain_lease_for(template_id) - Admin::PropagateServiceTemplate.propagate(Service.find_by(id: template_id)) + Admin::PropagateServiceTemplate.propagate(Integration.find_by(id: template_id)) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/prune_old_events_worker.rb b/app/workers/prune_old_events_worker.rb index 330de4c7cba..59d324bc573 100644 --- a/app/workers/prune_old_events_worker.rb +++ b/app/workers/prune_old_events_worker.rb @@ -2,6 +2,8 @@ class PruneOldEventsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/prune_web_hook_logs_worker.rb b/app/workers/prune_web_hook_logs_worker.rb index a8e81a24ecd..abfaabbf01d 100644 --- a/app/workers/prune_web_hook_logs_worker.rb +++ b/app/workers/prune_web_hook_logs_worker.rb @@ -4,6 +4,8 @@ # table. class PruneWebHookLogsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/purge_dependency_proxy_cache_worker.rb b/app/workers/purge_dependency_proxy_cache_worker.rb index b4c88592543..9f1ea8a6eb4 100644 --- a/app/workers/purge_dependency_proxy_cache_worker.rb +++ b/app/workers/purge_dependency_proxy_cache_worker.rb @@ -2,6 +2,8 @@ class PurgeDependencyProxyCacheWorker include ApplicationWorker + + sidekiq_options retry: 3 include Gitlab::Allowable idempotent! diff --git a/app/workers/rebase_worker.rb b/app/workers/rebase_worker.rb index ee9ae827bb6..664905eb9e5 100644 --- a/app/workers/rebase_worker.rb +++ b/app/workers/rebase_worker.rb @@ -5,6 +5,8 @@ class RebaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management weight 2 loggable_arguments 2 @@ -14,7 +16,7 @@ class RebaseWorker # rubocop:disable Scalability/IdempotentWorker merge_request = MergeRequest.find(merge_request_id) MergeRequests::RebaseService - .new(merge_request.source_project, current_user) + .new(project: merge_request.source_project, current_user: current_user) .execute(merge_request, skip_ci: skip_ci) end end diff --git a/app/workers/releases/create_evidence_worker.rb b/app/workers/releases/create_evidence_worker.rb index d22329216f9..bd790e8d0ee 100644 --- a/app/workers/releases/create_evidence_worker.rb +++ b/app/workers/releases/create_evidence_worker.rb @@ -4,7 +4,10 @@ module Releases class CreateEvidenceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :release_evidence + tags :exclude_from_kubernetes # pipeline_id is optional for backward compatibility with existing jobs # caller should always try to provide the pipeline and pass nil only diff --git a/app/workers/releases/manage_evidence_worker.rb b/app/workers/releases/manage_evidence_worker.rb index 8a925d22cea..88b6c4aea06 100644 --- a/app/workers/releases/manage_evidence_worker.rb +++ b/app/workers/releases/manage_evidence_worker.rb @@ -3,9 +3,12 @@ module Releases class ManageEvidenceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :release_evidence + tags :exclude_from_kubernetes def perform releases = Release.without_evidence.released_within_2hrs diff --git a/app/workers/remote_mirror_notification_worker.rb b/app/workers/remote_mirror_notification_worker.rb index 33f5002014d..39a7c0fc79d 100644 --- a/app/workers/remote_mirror_notification_worker.rb +++ b/app/workers/remote_mirror_notification_worker.rb @@ -3,6 +3,8 @@ class RemoteMirrorNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management weight 2 diff --git a/app/workers/remove_expired_group_links_worker.rb b/app/workers/remove_expired_group_links_worker.rb index 3f1a484f384..edf3a02cff5 100644 --- a/app/workers/remove_expired_group_links_worker.rb +++ b/app/workers/remove_expired_group_links_worker.rb @@ -2,6 +2,8 @@ class RemoveExpiredGroupLinksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :authentication_and_authorization diff --git a/app/workers/remove_expired_members_worker.rb b/app/workers/remove_expired_members_worker.rb index fc2ec047e1c..9940953207e 100644 --- a/app/workers/remove_expired_members_worker.rb +++ b/app/workers/remove_expired_members_worker.rb @@ -2,6 +2,8 @@ class RemoveExpiredMembersWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :authentication_and_authorization @@ -26,7 +28,7 @@ class RemoveExpiredMembersWorker # rubocop:disable Scalability/IdempotentWorker Users::DestroyService.new(nil).execute(expired_user, skip_authorization: true) end end - rescue => ex + rescue StandardError => ex logger.error("Expired Member ID=#{member.id} cannot be removed - #{ex}") end end diff --git a/app/workers/remove_unaccepted_member_invites_worker.rb b/app/workers/remove_unaccepted_member_invites_worker.rb index 4b75b43791e..c1f8e3881f1 100644 --- a/app/workers/remove_unaccepted_member_invites_worker.rb +++ b/app/workers/remove_unaccepted_member_invites_worker.rb @@ -2,9 +2,12 @@ class RemoveUnacceptedMemberInvitesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :authentication_and_authorization + tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/remove_unreferenced_lfs_objects_worker.rb b/app/workers/remove_unreferenced_lfs_objects_worker.rb index 76ab23ebbd5..b42883549ca 100644 --- a/app/workers/remove_unreferenced_lfs_objects_worker.rb +++ b/app/workers/remove_unreferenced_lfs_objects_worker.rb @@ -2,6 +2,8 @@ class RemoveUnreferencedLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/repository_archive_cache_worker.rb b/app/workers/repository_archive_cache_worker.rb index 84f61a60953..84cafba17cf 100644 --- a/app/workers/repository_archive_cache_worker.rb +++ b/app/workers/repository_archive_cache_worker.rb @@ -2,6 +2,8 @@ class RepositoryArchiveCacheWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index d47f738ccb0..a8744638d7b 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -3,6 +3,8 @@ module RepositoryCheck class BatchWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include RepositoryCheckQueue include ExclusiveLeaseGuard @@ -95,4 +97,4 @@ module RepositoryCheck end end -RepositoryCheck::BatchWorker.prepend_if_ee('::EE::RepositoryCheck::BatchWorker') +RepositoryCheck::BatchWorker.prepend_mod_with('RepositoryCheck::BatchWorker') diff --git a/app/workers/repository_check/clear_worker.rb b/app/workers/repository_check/clear_worker.rb index 1689b9bf251..bc19b42da1a 100644 --- a/app/workers/repository_check/clear_worker.rb +++ b/app/workers/repository_check/clear_worker.rb @@ -3,6 +3,8 @@ module RepositoryCheck class ClearWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include RepositoryCheckQueue # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/repository_check/dispatch_worker.rb b/app/workers/repository_check/dispatch_worker.rb index d7a145011fa..30734926765 100644 --- a/app/workers/repository_check/dispatch_worker.rb +++ b/app/workers/repository_check/dispatch_worker.rb @@ -3,6 +3,8 @@ module RepositoryCheck class DispatchWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb index d757b87c23a..a9a8201205e 100644 --- a/app/workers/repository_check/single_repository_worker.rb +++ b/app/workers/repository_check/single_repository_worker.rb @@ -3,6 +3,8 @@ module RepositoryCheck class SingleRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include RepositoryCheckQueue def perform(project_id) @@ -67,4 +69,4 @@ module RepositoryCheck end end -RepositoryCheck::SingleRepositoryWorker.prepend_if_ee('::EE::RepositoryCheck::SingleRepositoryWorker') +RepositoryCheck::SingleRepositoryWorker.prepend_mod_with('RepositoryCheck::SingleRepositoryWorker') diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index fc7999e7837..06a6f5b0600 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -2,6 +2,8 @@ class RepositoryForkWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ProjectStartImport include ProjectImportOptions diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 90764d7374d..0f86d55df22 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -63,4 +63,4 @@ class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker end end -RepositoryImportWorker.prepend_if_ee('EE::RepositoryImportWorker') +RepositoryImportWorker.prepend_mod_with('RepositoryImportWorker') diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb index 5e632b1b1ca..48158cda857 100644 --- a/app/workers/repository_remove_remote_worker.rb +++ b/app/workers/repository_remove_remote_worker.rb @@ -2,6 +2,8 @@ class RepositoryRemoveRemoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExclusiveLeaseGuard feature_category :source_code_management diff --git a/app/workers/requests_profiles_worker.rb b/app/workers/requests_profiles_worker.rb index 106f04d9409..35c18177a81 100644 --- a/app/workers/requests_profiles_worker.rb +++ b/app/workers/requests_profiles_worker.rb @@ -2,6 +2,8 @@ class RequestsProfilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb index 605dd624260..553153848c7 100644 --- a/app/workers/run_pipeline_schedule_worker.rb +++ b/app/workers/run_pipeline_schedule_worker.rb @@ -2,6 +2,8 @@ class RunPipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_creation @@ -25,7 +27,7 @@ class RunPipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker .execute!(:schedule, ignore_skip_ci: true, save_on_errors: false, schedule: schedule) rescue Ci::CreatePipelineService::CreateError # no-op. This is a user operation error such as corrupted .gitlab-ci.yml. - rescue => e + rescue StandardError => e error(schedule, e) end diff --git a/app/workers/schedule_merge_request_cleanup_refs_worker.rb b/app/workers/schedule_merge_request_cleanup_refs_worker.rb index 967032f99e5..b5ea5298879 100644 --- a/app/workers/schedule_merge_request_cleanup_refs_worker.rb +++ b/app/workers/schedule_merge_request_cleanup_refs_worker.rb @@ -2,9 +2,12 @@ class ScheduleMergeRequestCleanupRefsWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :code_review + tags :exclude_from_kubernetes idempotent! # Based on existing data, MergeRequestCleanupRefsWorker can run 3 jobs per diff --git a/app/workers/schedule_migrate_external_diffs_worker.rb b/app/workers/schedule_migrate_external_diffs_worker.rb index 70e4d56562b..ecafe8f5e7d 100644 --- a/app/workers/schedule_migrate_external_diffs_worker.rb +++ b/app/workers/schedule_migrate_external_diffs_worker.rb @@ -2,6 +2,8 @@ class ScheduleMigrateExternalDiffsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext: # This schedules the `MigrateExternalDiffsWorker` # issue for adding context: https://gitlab.com/gitlab-org/gitlab/issues/202100 diff --git a/app/workers/self_monitoring_project_create_worker.rb b/app/workers/self_monitoring_project_create_worker.rb index 8177efb1683..9dc3bb855fb 100644 --- a/app/workers/self_monitoring_project_create_worker.rb +++ b/app/workers/self_monitoring_project_create_worker.rb @@ -2,6 +2,8 @@ class SelfMonitoringProjectCreateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExclusiveLeaseGuard include SelfMonitoringProjectWorker diff --git a/app/workers/self_monitoring_project_delete_worker.rb b/app/workers/self_monitoring_project_delete_worker.rb index 4fa05d71de5..c155c57dec7 100644 --- a/app/workers/self_monitoring_project_delete_worker.rb +++ b/app/workers/self_monitoring_project_delete_worker.rb @@ -2,6 +2,8 @@ class SelfMonitoringProjectDeleteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ExclusiveLeaseGuard include SelfMonitoringProjectWorker diff --git a/app/workers/service_desk_email_receiver_worker.rb b/app/workers/service_desk_email_receiver_worker.rb index 8649034445c..cfe63e059bb 100644 --- a/app/workers/service_desk_email_receiver_worker.rb +++ b/app/workers/service_desk_email_receiver_worker.rb @@ -3,13 +3,14 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - def perform(raw) - return unless ::Gitlab::ServiceDeskEmail.enabled? - - begin - Gitlab::Email::ServiceDeskReceiver.new(raw).execute - rescue => e - handle_failure(raw, e) - end + feature_category :service_desk + sidekiq_options retry: 3 + + def should_perform? + ::Gitlab::ServiceDeskEmail.enabled? + end + + def receiver + @receiver ||= Gitlab::Email::ServiceDeskReceiver.new(raw) end end diff --git a/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb b/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb index ec3d9dbdf97..88b060a454a 100644 --- a/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb +++ b/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb @@ -4,6 +4,8 @@ module Snippets class ScheduleBulkRepositoryShardMovesWorker include ApplicationWorker + sidekiq_options retry: 3 + idempotent! feature_category :gitaly urgency :throttled diff --git a/app/workers/snippets/update_repository_storage_worker.rb b/app/workers/snippets/update_repository_storage_worker.rb index 83b655e9986..ffb01e2623b 100644 --- a/app/workers/snippets/update_repository_storage_worker.rb +++ b/app/workers/snippets/update_repository_storage_worker.rb @@ -5,6 +5,8 @@ module Snippets extend ::Gitlab::Utils::Override include ::UpdateRepositoryStorageWorker + sidekiq_options retry: 3 + private override :find_repository_storage_move diff --git a/app/workers/ssh_keys/expired_notification_worker.rb b/app/workers/ssh_keys/expired_notification_worker.rb index ab6d1998773..9d5143fe655 100644 --- a/app/workers/ssh_keys/expired_notification_worker.rb +++ b/app/workers/ssh_keys/expired_notification_worker.rb @@ -3,15 +3,19 @@ module SshKeys class ExpiredNotificationWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :compliance_management + tags :exclude_from_kubernetes idempotent! def perform return unless ::Feature.enabled?(:ssh_key_expiration_email_notification, default_enabled: :yaml) - User.with_ssh_key_expired_today.find_each do |user| + # rubocop:disable CodeReuse/ActiveRecord + User.with_ssh_key_expired_today.find_each(batch_size: 10_000) do |user| with_context(user: user) do Gitlab::AppLogger.info "#{self.class}: Notifying User #{user.id} about expired ssh key(s)" @@ -19,6 +23,7 @@ module SshKeys Keys::ExpiryNotificationService.new(user, { keys: keys, expiring_soon: false }).execute end + # rubocop:enable CodeReuse/ActiveRecord end end end diff --git a/app/workers/ssh_keys/expiring_soon_notification_worker.rb b/app/workers/ssh_keys/expiring_soon_notification_worker.rb index 3214cd7a242..1ec655b5cf5 100644 --- a/app/workers/ssh_keys/expiring_soon_notification_worker.rb +++ b/app/workers/ssh_keys/expiring_soon_notification_worker.rb @@ -3,15 +3,19 @@ module SshKeys class ExpiringSoonNotificationWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :compliance_management + tags :exclude_from_kubernetes idempotent! def perform return unless ::Feature.enabled?(:ssh_key_expiration_email_notification, default_enabled: :yaml) - User.with_ssh_key_expiring_soon.find_each do |user| + # rubocop:disable CodeReuse/ActiveRecord + User.with_ssh_key_expiring_soon.find_each(batch_size: 10_000) do |user| with_context(user: user) do Gitlab::AppLogger.info "#{self.class}: Notifying User #{user.id} about expiring soon ssh key(s)" @@ -20,6 +24,7 @@ module SshKeys Keys::ExpiryNotificationService.new(user, { keys: keys, expiring_soon: true }).execute end end + # rubocop:enable CodeReuse/ActiveRecord end end end diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb index 20db19536c3..e206a51a417 100644 --- a/app/workers/stage_update_worker.rb +++ b/app/workers/stage_update_worker.rb @@ -2,6 +2,8 @@ class StageUpdateWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index bd721df73c6..6b9f90ce1fc 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -2,6 +2,8 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :continuous_integration @@ -73,7 +75,7 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker Gitlab::OptimisticLocking.retry_lock(build, 3, name: 'stuck_ci_jobs_worker_drop_build') do |b| b.drop(reason) end - rescue => ex + rescue StandardError => ex build.doom! track_exception_for_build(ex, build) diff --git a/app/workers/stuck_export_jobs_worker.rb b/app/workers/stuck_export_jobs_worker.rb index 6d8d60d2fc0..398f2c915a9 100644 --- a/app/workers/stuck_export_jobs_worker.rb +++ b/app/workers/stuck_export_jobs_worker.rb @@ -3,6 +3,8 @@ # rubocop:disable Scalability/IdempotentWorker class StuckExportJobsWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker updates export states inline and does not schedule # other jobs. diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb index bea9d67b3e8..e50b218e1f6 100644 --- a/app/workers/stuck_merge_jobs_worker.rb +++ b/app/workers/stuck_merge_jobs_worker.rb @@ -2,6 +2,8 @@ class StuckMergeJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :code_review diff --git a/app/workers/system_hook_push_worker.rb b/app/workers/system_hook_push_worker.rb index ff1f2baf058..8c801f2bed8 100644 --- a/app/workers/system_hook_push_worker.rb +++ b/app/workers/system_hook_push_worker.rb @@ -3,6 +3,8 @@ class SystemHookPushWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management def perform(push_data, hook_id) diff --git a/app/workers/todos_destroyer/confidential_issue_worker.rb b/app/workers/todos_destroyer/confidential_issue_worker.rb index b29d4168162..8a43ea3c2e0 100644 --- a/app/workers/todos_destroyer/confidential_issue_worker.rb +++ b/app/workers/todos_destroyer/confidential_issue_worker.rb @@ -3,6 +3,8 @@ module TodosDestroyer class ConfidentialIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include TodosDestroyerQueue def perform(issue_id = nil, project_id = nil) diff --git a/app/workers/todos_destroyer/destroyed_issuable_worker.rb b/app/workers/todos_destroyer/destroyed_issuable_worker.rb index 6ca1959ff34..a3a8147095e 100644 --- a/app/workers/todos_destroyer/destroyed_issuable_worker.rb +++ b/app/workers/todos_destroyer/destroyed_issuable_worker.rb @@ -3,8 +3,12 @@ module TodosDestroyer class DestroyedIssuableWorker include ApplicationWorker + + sidekiq_options retry: 3 include TodosDestroyerQueue + tags :exclude_from_kubernetes + idempotent! def perform(target_id, target_type) diff --git a/app/workers/todos_destroyer/entity_leave_worker.rb b/app/workers/todos_destroyer/entity_leave_worker.rb index 4996456dc91..166d8701f7a 100644 --- a/app/workers/todos_destroyer/entity_leave_worker.rb +++ b/app/workers/todos_destroyer/entity_leave_worker.rb @@ -3,6 +3,8 @@ module TodosDestroyer class EntityLeaveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include TodosDestroyerQueue loggable_arguments 2 diff --git a/app/workers/todos_destroyer/group_private_worker.rb b/app/workers/todos_destroyer/group_private_worker.rb index a1943bee2ec..30d1f74fb28 100644 --- a/app/workers/todos_destroyer/group_private_worker.rb +++ b/app/workers/todos_destroyer/group_private_worker.rb @@ -3,6 +3,8 @@ module TodosDestroyer class GroupPrivateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include TodosDestroyerQueue def perform(group_id) diff --git a/app/workers/todos_destroyer/private_features_worker.rb b/app/workers/todos_destroyer/private_features_worker.rb index 6e55467234a..d6a4260a464 100644 --- a/app/workers/todos_destroyer/private_features_worker.rb +++ b/app/workers/todos_destroyer/private_features_worker.rb @@ -3,6 +3,8 @@ module TodosDestroyer class PrivateFeaturesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include TodosDestroyerQueue def perform(project_id, user_id = nil) diff --git a/app/workers/todos_destroyer/project_private_worker.rb b/app/workers/todos_destroyer/project_private_worker.rb index 2a06edc666e..c4fed03f11a 100644 --- a/app/workers/todos_destroyer/project_private_worker.rb +++ b/app/workers/todos_destroyer/project_private_worker.rb @@ -3,6 +3,8 @@ module TodosDestroyer class ProjectPrivateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include TodosDestroyerQueue def perform(project_id) diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb index 5876cfb1fe7..8322110b753 100644 --- a/app/workers/trending_projects_worker.rb +++ b/app/workers/trending_projects_worker.rb @@ -2,6 +2,8 @@ class TrendingProjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :source_code_management diff --git a/app/workers/update_container_registry_info_worker.rb b/app/workers/update_container_registry_info_worker.rb index 14a816f25ef..cf08c650d0d 100644 --- a/app/workers/update_container_registry_info_worker.rb +++ b/app/workers/update_container_registry_info_worker.rb @@ -2,6 +2,8 @@ class UpdateContainerRegistryInfoWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :container_registry diff --git a/app/workers/update_external_pull_requests_worker.rb b/app/workers/update_external_pull_requests_worker.rb index e916331ae82..ee47cbd6523 100644 --- a/app/workers/update_external_pull_requests_worker.rb +++ b/app/workers/update_external_pull_requests_worker.rb @@ -3,6 +3,8 @@ class UpdateExternalPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management weight 3 loggable_arguments 2 diff --git a/app/workers/update_head_pipeline_for_merge_request_worker.rb b/app/workers/update_head_pipeline_for_merge_request_worker.rb index 63d11d33283..f1dd250f432 100644 --- a/app/workers/update_head_pipeline_for_merge_request_worker.rb +++ b/app/workers/update_head_pipeline_for_merge_request_worker.rb @@ -2,6 +2,8 @@ class UpdateHeadPipelineForMergeRequestWorker include ApplicationWorker + + sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_processing diff --git a/app/workers/update_highest_role_worker.rb b/app/workers/update_highest_role_worker.rb index 952f1e511ea..cecf3f99b50 100644 --- a/app/workers/update_highest_role_worker.rb +++ b/app/workers/update_highest_role_worker.rb @@ -3,6 +3,8 @@ class UpdateHighestRoleWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :utilization urgency :high weight 2 @@ -15,7 +17,7 @@ class UpdateHighestRoleWorker return unless user.present? - if user.active? && user.user_type.nil? && !user.internal? + if user.active? && user.human? && !user.internal? Users::UpdateHighestMemberRoleService.new(user).execute else UserHighestRole.where(user_id: user_id).delete_all diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index 46cb32e7f08..6f86a7e7e2f 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -3,6 +3,8 @@ class UpdateMergeRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :code_review urgency :high worker_resource_boundary :cpu @@ -17,7 +19,7 @@ class UpdateMergeRequestsWorker # rubocop:disable Scalability/IdempotentWorker user = User.find_by(id: user_id) return unless user - MergeRequests::RefreshService.new(project, user).execute(oldrev, newrev, ref) + MergeRequests::RefreshService.new(project: project, current_user: user).execute(oldrev, newrev, ref) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/update_project_statistics_worker.rb b/app/workers/update_project_statistics_worker.rb index 336877d9f57..c93c32f4e75 100644 --- a/app/workers/update_project_statistics_worker.rb +++ b/app/workers/update_project_statistics_worker.rb @@ -4,6 +4,8 @@ class UpdateProjectStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management # project_id - The ID of the project for which to flush the cache. diff --git a/app/workers/upload_checksum_worker.rb b/app/workers/upload_checksum_worker.rb index ce43b56bbd8..765e3a63e75 100644 --- a/app/workers/upload_checksum_worker.rb +++ b/app/workers/upload_checksum_worker.rb @@ -3,6 +3,8 @@ class UploadChecksumWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :geo_replication def perform(upload_id) diff --git a/app/workers/user_status_cleanup/batch_worker.rb b/app/workers/user_status_cleanup/batch_worker.rb index 0c1087cc4d2..f46b4119f9b 100644 --- a/app/workers/user_status_cleanup/batch_worker.rb +++ b/app/workers/user_status_cleanup/batch_worker.rb @@ -4,11 +4,14 @@ module UserStatusCleanup # This worker will run every minute to look for user status records to clean up. class BatchWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext include CronjobQueue # rubocop:enable Scalability/CronWorkerContext feature_category :users + tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/users/create_statistics_worker.rb b/app/workers/users/create_statistics_worker.rb index fb1b192577f..e44039f2016 100644 --- a/app/workers/users/create_statistics_worker.rb +++ b/app/workers/users/create_statistics_worker.rb @@ -3,6 +3,8 @@ module Users class CreateStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue diff --git a/app/workers/users/deactivate_dormant_users_worker.rb b/app/workers/users/deactivate_dormant_users_worker.rb new file mode 100644 index 00000000000..e583823312f --- /dev/null +++ b/app/workers/users/deactivate_dormant_users_worker.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Users + class DeactivateDormantUsersWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + include CronjobQueue + + feature_category :utilization + tags :exclude_from_kubernetes + + NUMBER_OF_BATCHES = 50 + BATCH_SIZE = 200 + PAUSE_SECONDS = 0.25 + + def perform + return if Gitlab.com? + + return unless ::Gitlab::CurrentSettings.current_application_settings.deactivate_dormant_users + + with_context(caller_id: self.class.name.to_s) do + NUMBER_OF_BATCHES.times do + result = User.connection.execute(update_query) + + break if result.cmd_tuples == 0 + + sleep(PAUSE_SECONDS) + end + end + end + + private + + def update_query + <<~SQL + UPDATE "users" + SET "state" = 'deactivated' + WHERE "users"."id" IN ( + (#{users.dormant.to_sql}) + UNION + (#{users.with_no_activity.to_sql}) + LIMIT #{BATCH_SIZE} + ) + SQL + end + + def users + User.select(:id).limit(BATCH_SIZE) + end + end +end diff --git a/app/workers/users/update_open_issue_count_worker.rb b/app/workers/users/update_open_issue_count_worker.rb new file mode 100644 index 00000000000..d9e313d53df --- /dev/null +++ b/app/workers/users/update_open_issue_count_worker.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Users + class UpdateOpenIssueCountWorker + include ApplicationWorker + + feature_category :users + tags :exclude_from_kubernetes + idempotent! + + def perform(target_user_ids) + target_user_ids = Array.wrap(target_user_ids) + + raise ArgumentError, 'No target user ID provided' if target_user_ids.empty? + + target_users = User.id_in(target_user_ids) + raise ArgumentError, 'No valid target user ID provided' if target_users.empty? + + target_users.each do |user| + Users::UpdateAssignedOpenIssueCountService.new(target_user: user).execute + end + rescue StandardError => exception + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + end + end +end diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb index 2e3feb1a4d1..525a72e02ef 100644 --- a/app/workers/wait_for_cluster_creation_worker.rb +++ b/app/workers/wait_for_cluster_creation_worker.rb @@ -2,6 +2,8 @@ class WaitForClusterCreationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + + sidekiq_options retry: 3 include ClusterQueue worker_has_external_dependencies! diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index 5230f3bfa1f..dffab61dd0e 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true -class WebHookWorker # rubocop:disable Scalability/IdempotentWorker +# Worker cannot be idempotent: https://gitlab.com/gitlab-org/gitlab/-/issues/218559 +# rubocop:disable Scalability/IdempotentWorker +class WebHookWorker include ApplicationWorker feature_category :integrations @@ -16,3 +18,4 @@ class WebHookWorker # rubocop:disable Scalability/IdempotentWorker WebHookService.new(hook, data, hook_name).execute end end +# rubocop:enable Scalability/IdempotentWorker diff --git a/app/workers/web_hooks/destroy_worker.rb b/app/workers/web_hooks/destroy_worker.rb index 13a5a7bf1e6..c1886576c41 100644 --- a/app/workers/web_hooks/destroy_worker.rb +++ b/app/workers/web_hooks/destroy_worker.rb @@ -4,7 +4,10 @@ module WebHooks class DestroyWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :integrations + tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/wikis/git_garbage_collect_worker.rb b/app/workers/wikis/git_garbage_collect_worker.rb index 1b455c50618..f34d3be51d2 100644 --- a/app/workers/wikis/git_garbage_collect_worker.rb +++ b/app/workers/wikis/git_garbage_collect_worker.rb @@ -5,6 +5,8 @@ module Wikis extend ::Gitlab::Utils::Override include GitGarbageCollectMethods + tags :exclude_from_kubernetes + private override :find_resource diff --git a/app/workers/x509_certificate_revoke_worker.rb b/app/workers/x509_certificate_revoke_worker.rb index abd0e5eefa7..cbf9fbb7525 100644 --- a/app/workers/x509_certificate_revoke_worker.rb +++ b/app/workers/x509_certificate_revoke_worker.rb @@ -3,6 +3,8 @@ class X509CertificateRevokeWorker include ApplicationWorker + sidekiq_options retry: 3 + feature_category :source_code_management idempotent! diff --git a/app/workers/x509_issuer_crl_check_worker.rb b/app/workers/x509_issuer_crl_check_worker.rb index 5fc92da803c..d66ad6c1c15 100644 --- a/app/workers/x509_issuer_crl_check_worker.rb +++ b/app/workers/x509_issuer_crl_check_worker.rb @@ -2,6 +2,8 @@ class X509IssuerCrlCheckWorker include ApplicationWorker + + sidekiq_options retry: 3 include CronjobQueue feature_category :source_code_management |