diff options
Diffstat (limited to 'app/workers')
105 files changed, 293 insertions, 542 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f326ae0dec8..955674b52a4 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -83,8 +83,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: - - :exclude_from_gitlab_com + :tags: [] - :name: chaos:chaos_db_spin :worker_name: Chaos::DbSpinWorker :feature_category: :not_owned @@ -93,8 +92,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: - - :exclude_from_gitlab_com + :tags: [] - :name: chaos:chaos_kill :worker_name: Chaos::KillWorker :feature_category: :not_owned @@ -103,8 +101,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: - - :exclude_from_gitlab_com + :tags: [] - :name: chaos:chaos_leak_mem :worker_name: Chaos::LeakMemWorker :feature_category: :not_owned @@ -113,8 +110,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: - - :exclude_from_gitlab_com + :tags: [] - :name: chaos:chaos_sleep :worker_name: Chaos::SleepWorker :feature_category: :not_owned @@ -123,8 +119,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: - - :exclude_from_gitlab_com + :tags: [] - :name: container_repository:cleanup_container_repository :worker_name: CleanupContainerRepositoryWorker :feature_category: :container_registry @@ -142,8 +137,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: container_repository:delete_container_repository :worker_name: DeleteContainerRepositoryWorker :feature_category: :container_registry @@ -170,8 +164,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:authorized_project_update_periodic_recalculate :worker_name: AuthorizedProjectUpdate::PeriodicRecalculateWorker :feature_category: :source_code_management @@ -207,8 +200,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:ci_platform_metrics_update_cron :worker_name: CiPlatformMetricsUpdateCronWorker :feature_category: :continuous_integration @@ -226,8 +218,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:container_expiration_policy :worker_name: ContainerExpirationPolicyWorker :feature_category: :container_registry @@ -245,8 +236,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:database_drop_detached_partitions :worker_name: Database::DropDetachedPartitionsWorker :feature_category: :database @@ -345,8 +335,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:metrics_dashboard_schedule_annotations_prune :worker_name: Metrics::Dashboard::ScheduleAnnotationsPruneWorker :feature_category: :metrics @@ -364,8 +353,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:namespaces_prune_aggregation_schedules :worker_name: Namespaces::PruneAggregationSchedulesWorker :feature_category: :source_code_management @@ -383,8 +371,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:pages_domain_removal_cron :worker_name: PagesDomainRemovalCronWorker :feature_category: :pages @@ -429,8 +416,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:personal_access_tokens_expiring :worker_name: PersonalAccessTokens::ExpiringWorker :feature_category: :authentication_and_authorization @@ -466,8 +452,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:remove_expired_group_links :worker_name: RemoveExpiredGroupLinksWorker :feature_category: :authentication_and_authorization @@ -494,8 +479,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:remove_unreferenced_lfs_objects :worker_name: RemoveUnreferencedLfsObjectsWorker :feature_category: :git_lfs @@ -540,8 +524,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:schedule_migrate_external_diffs :worker_name: ScheduleMigrateExternalDiffsWorker :feature_category: :code_review @@ -559,8 +542,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:ssh_keys_expiring_soon_notification :worker_name: SshKeys::ExpiringSoonNotificationWorker :feature_category: :compliance_management @@ -569,8 +551,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:stuck_ci_jobs :worker_name: StuckCiJobsWorker :feature_category: :continuous_integration @@ -624,8 +605,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:users_create_statistics :worker_name: Users::CreateStatisticsWorker :feature_category: :users @@ -643,8 +623,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: cronjob:x509_issuer_crl_check :worker_name: X509IssuerCrlCheckWorker :feature_category: :source_code_management @@ -671,25 +650,6 @@ :resource_boundary: :unknown :weight: 3 :idempotent: - :tags: - - :exclude_from_kubernetes -- :name: deployment:deployments_finished - :worker_name: Deployments::FinishedWorker - :feature_category: :continuous_delivery - :has_external_dependencies: - :urgency: :low - :resource_boundary: :cpu - :weight: 3 - :idempotent: - :tags: [] -- :name: deployment:deployments_forward_deployment - :worker_name: Deployments::ForwardDeploymentWorker - :feature_category: :continuous_delivery - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 3 - :idempotent: :tags: [] - :name: deployment:deployments_hooks :worker_name: Deployments::HooksWorker @@ -709,15 +669,6 @@ :weight: 3 :idempotent: true :tags: [] -- :name: deployment:deployments_success - :worker_name: Deployments::SuccessWorker - :feature_category: :continuous_delivery - :has_external_dependencies: - :urgency: :low - :resource_boundary: :cpu - :weight: 3 - :idempotent: - :tags: [] - :name: deployment:deployments_update_environment :worker_name: Deployments::UpdateEnvironmentWorker :feature_category: :continuous_delivery @@ -924,8 +875,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: github_importer:github_import_import_pull_request_review :worker_name: Gitlab::GithubImport::ImportPullRequestReviewWorker :feature_category: :importers @@ -934,8 +884,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: github_importer:github_import_refresh_import_jid :worker_name: Gitlab::GithubImport::RefreshImportJidWorker :feature_category: :importers @@ -1007,8 +956,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: github_importer:github_import_stage_import_pull_requests_reviews :worker_name: Gitlab::GithubImport::Stage::ImportPullRequestsReviewsWorker :feature_category: :importers @@ -1017,8 +965,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: github_importer:github_import_stage_import_repository :worker_name: Gitlab::GithubImport::Stage::ImportRepositoryWorker :feature_category: :importers @@ -1037,7 +984,6 @@ :weight: 1 :idempotent: :tags: - - :exclude_from_gitlab_com - :needs_own_queue - :name: hashed_storage:hashed_storage_project_migrate :worker_name: HashedStorage::ProjectMigrateWorker @@ -1048,7 +994,6 @@ :weight: 1 :idempotent: :tags: - - :exclude_from_gitlab_com - :needs_own_queue - :name: hashed_storage:hashed_storage_project_rollback :worker_name: HashedStorage::ProjectRollbackWorker @@ -1059,7 +1004,6 @@ :weight: 1 :idempotent: :tags: - - :exclude_from_gitlab_com - :needs_own_queue - :name: hashed_storage:hashed_storage_rollbacker :worker_name: HashedStorage::RollbackerWorker @@ -1070,7 +1014,6 @@ :weight: 1 :idempotent: :tags: - - :exclude_from_gitlab_com - :needs_own_queue - :name: incident_management:clusters_applications_check_prometheus_health :worker_name: Clusters::Applications::CheckPrometheusHealthWorker @@ -1089,8 +1032,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: incident_management:incident_management_pager_duty_process_incident :worker_name: IncidentManagement::PagerDuty::ProcessIncidentWorker :feature_category: :incident_management @@ -1144,8 +1086,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: jira_connect:jira_connect_sync_deployments :worker_name: JiraConnect::SyncDeploymentsWorker :feature_category: :integrations @@ -1154,8 +1095,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: jira_connect:jira_connect_sync_feature_flags :worker_name: JiraConnect::SyncFeatureFlagsWorker :feature_category: :integrations @@ -1164,8 +1104,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: jira_connect:jira_connect_sync_merge_request :worker_name: JiraConnect::SyncMergeRequestWorker :feature_category: :integrations @@ -1183,8 +1122,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: jira_importer:jira_import_advance_stage :worker_name: Gitlab::JiraImport::AdvanceStageWorker :feature_category: :importers @@ -1346,8 +1284,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: package_repositories:packages_go_sync_packages :worker_name: Packages::Go::SyncPackagesWorker :feature_category: :package_registry @@ -1356,8 +1293,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: package_repositories:packages_helm_extraction :worker_name: Packages::Helm::ExtractionWorker :feature_category: :package_registry @@ -1375,8 +1311,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: package_repositories:packages_nuget_extraction :worker_name: Packages::Nuget::ExtractionWorker :feature_category: :package_registry @@ -1394,8 +1329,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pipeline_background:archive_trace :worker_name: ArchiveTraceWorker :feature_category: :continuous_integration @@ -1449,8 +1383,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pipeline_background:ci_pipeline_success_unlock_artifacts :worker_name: Ci::PipelineSuccessUnlockArtifactsWorker :feature_category: :continuous_integration @@ -1477,8 +1410,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pipeline_cache:expire_job_cache :worker_name: ExpireJobCacheWorker :feature_category: :continuous_integration @@ -1486,7 +1418,7 @@ :urgency: :high :resource_boundary: :unknown :weight: 3 - :idempotent: true + :idempotent: :tags: [] - :name: pipeline_cache:expire_pipeline_cache :worker_name: ExpirePipelineCacheWorker @@ -1497,6 +1429,15 @@ :weight: 3 :idempotent: :tags: [] +- :name: pipeline_creation:ci_external_pull_requests_create_pipeline + :worker_name: Ci::ExternalPullRequests::CreatePipelineWorker + :feature_category: :pipeline_authoring + :has_external_dependencies: + :urgency: :high + :resource_boundary: :cpu + :weight: 4 + :idempotent: + :tags: [] - :name: pipeline_creation:create_pipeline :worker_name: CreatePipelineWorker :feature_category: :continuous_integration @@ -1541,8 +1482,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pipeline_default:ci_merge_requests_add_todo_when_build_fails :worker_name: Ci::MergeRequests::AddTodoWhenBuildFailsWorker :feature_category: :continuous_integration @@ -1551,8 +1491,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pipeline_default:ci_pipeline_bridge_status :worker_name: Ci::PipelineBridgeStatusWorker :feature_category: :continuous_integration @@ -1742,6 +1681,15 @@ :weight: 1 :idempotent: :tags: [] +- :name: todos_destroyer:todos_destroyer_destroyed_designs + :worker_name: TodosDestroyer::DestroyedDesignsWorker + :feature_category: :issue_tracking + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: todos_destroyer:todos_destroyer_destroyed_issuable :worker_name: TodosDestroyer::DestroyedIssuableWorker :feature_category: :issue_tracking @@ -1750,8 +1698,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: todos_destroyer:todos_destroyer_entity_leave :worker_name: TodosDestroyer::EntityLeaveWorker :feature_category: :issue_tracking @@ -1823,8 +1770,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: approve_blocked_pending_approval_users :worker_name: ApproveBlockedPendingApprovalUsersWorker :feature_category: :users @@ -1833,8 +1779,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: authorized_keys :worker_name: AuthorizedKeysWorker :feature_category: :source_code_management @@ -1870,8 +1815,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: bulk_imports_entity :worker_name: BulkImports::EntityWorker :feature_category: :importers @@ -1880,8 +1824,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: bulk_imports_export_request :worker_name: BulkImports::ExportRequestWorker :feature_category: :importers @@ -1899,8 +1842,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: bulk_imports_relation_export :worker_name: BulkImports::RelationExportWorker :feature_category: :importers @@ -1909,8 +1851,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: chat_notification :worker_name: ChatNotificationWorker :feature_category: :chatops @@ -1928,8 +1869,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: create_commit_signature :worker_name: CreateCommitSignatureWorker :feature_category: :source_code_management @@ -2001,8 +1941,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: design_management_new_version :worker_name: DesignManagement::NewVersionWorker :feature_category: :design_management @@ -2020,8 +1959,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: detect_repository_languages :worker_name: DetectRepositoryLanguagesWorker :feature_category: :source_code_management @@ -2039,8 +1977,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: disallow_two_factor_for_subgroups :worker_name: DisallowTwoFactorForSubgroupsWorker :feature_category: :subgroups @@ -2049,8 +1986,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: email_receiver :worker_name: EmailReceiverWorker :feature_category: :issue_tracking @@ -2070,6 +2006,15 @@ :weight: 2 :idempotent: :tags: [] +- :name: environments_auto_stop + :worker_name: Environments::AutoStopWorker + :feature_category: :continuous_delivery + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: environments_canary_ingress_update :worker_name: Environments::CanaryIngress::UpdateWorker :feature_category: :continuous_delivery @@ -2078,8 +2023,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: error_tracking_issue_link :worker_name: ErrorTrackingIssueLinkWorker :feature_category: :error_tracking @@ -2097,8 +2041,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: expire_build_instance_artifacts :worker_name: ExpireBuildInstanceArtifactsWorker :feature_category: :continuous_integration @@ -2143,8 +2086,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: github_import_advance_stage :worker_name: Gitlab::GithubImport::AdvanceStageWorker :feature_category: :importers @@ -2162,8 +2104,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: gitlab_shell :worker_name: GitlabShellWorker :feature_category: :source_code_management @@ -2181,9 +2122,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :requires_disk_io - - :exclude_from_kubernetes + :tags: [] - :name: group_export :worker_name: GroupExportWorker :feature_category: :importers @@ -2273,8 +2212,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: mailers :worker_name: ActionMailer::MailDeliveryJob :feature_category: :issue_tracking @@ -2301,8 +2239,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: merge_request_mergeability_check :worker_name: MergeRequestMergeabilityCheckWorker :feature_category: :code_review @@ -2356,8 +2293,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: migrate_external_diffs :worker_name: MigrateExternalDiffsWorker :feature_category: :code_review @@ -2384,8 +2320,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: namespaces_onboarding_pipeline_created :worker_name: Namespaces::OnboardingPipelineCreatedWorker :feature_category: :subgroups @@ -2394,8 +2329,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: namespaces_onboarding_progress :worker_name: Namespaces::OnboardingProgressWorker :feature_category: :product_analytics @@ -2404,8 +2338,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: namespaces_onboarding_user_added :worker_name: Namespaces::OnboardingUserAddedWorker :feature_category: :users @@ -2414,8 +2347,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: new_issue :worker_name: NewIssueWorker :feature_category: :issue_tracking @@ -2451,8 +2383,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pages :worker_name: PagesWorker :feature_category: :pages @@ -2461,9 +2392,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: - :tags: - - :requires_disk_io - - :exclude_from_kubernetes + :tags: [] - :name: pages_domain_ssl_renewal :worker_name: PagesDomainSslRenewalWorker :feature_category: :pages @@ -2472,9 +2401,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :requires_disk_io - - :exclude_from_kubernetes + :tags: [] - :name: pages_domain_verification :worker_name: PagesDomainVerificationWorker :feature_category: :pages @@ -2483,9 +2410,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :requires_disk_io - - :exclude_from_kubernetes + :tags: [] - :name: pages_remove :worker_name: PagesRemoveWorker :feature_category: :pages @@ -2494,8 +2419,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pages_transfer :worker_name: PagesTransferWorker :feature_category: :pages @@ -2504,8 +2428,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: pages_update_configuration :worker_name: PagesUpdateConfigurationWorker :feature_category: :pages @@ -2514,8 +2437,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: phabricator_import_import_tasks :worker_name: Gitlab::PhabricatorImport::ImportTasksWorker :feature_category: :importers @@ -2532,7 +2454,7 @@ :urgency: :high :resource_boundary: :cpu :weight: 5 - :idempotent: + :idempotent: true :tags: [] - :name: process_commit :worker_name: ProcessCommitWorker @@ -2569,9 +2491,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :requires_disk_io - - :exclude_from_kubernetes + :tags: [] - :name: project_export :worker_name: ProjectExportWorker :feature_category: :importers @@ -2598,8 +2518,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: projects_post_creation :worker_name: Projects::PostCreationWorker :feature_category: :source_code_management @@ -2608,8 +2527,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: projects_schedule_bulk_repository_shard_moves :worker_name: Projects::ScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly @@ -2654,8 +2572,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: propagate_integration_inherit :worker_name: PropagateIntegrationInheritWorker :feature_category: :integrations @@ -2664,8 +2581,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: propagate_integration_inherit_descendant :worker_name: PropagateIntegrationInheritDescendantWorker :feature_category: :integrations @@ -2674,8 +2590,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: propagate_integration_project :worker_name: PropagateIntegrationProjectWorker :feature_category: :integrations @@ -2684,8 +2599,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: propagate_service_template :worker_name: PropagateServiceTemplateWorker :feature_category: :integrations @@ -2721,8 +2635,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: remote_mirror_notification :worker_name: RemoteMirrorNotificationWorker :feature_category: :source_code_management @@ -2799,7 +2712,7 @@ :worker_name: ServiceDeskEmailReceiverWorker :feature_category: :service_desk :has_external_dependencies: - :urgency: :low + :urgency: :high :resource_boundary: :unknown :weight: 1 :idempotent: @@ -2894,8 +2807,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: web_hooks_log_execution :worker_name: WebHooks::LogExecutionWorker :feature_category: :integrations @@ -2913,8 +2825,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :exclude_from_kubernetes + :tags: [] - :name: x509_certificate_revoke :worker_name: X509CertificateRevokeWorker :feature_category: :source_code_management diff --git a/app/workers/analytics/usage_trends/count_job_trigger_worker.rb b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb index 41ef75ac20a..dae576a6521 100644 --- a/app/workers/analytics/usage_trends/count_job_trigger_worker.rb +++ b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb @@ -13,7 +13,6 @@ module Analytics DEFAULT_DELAY = 3.minutes.freeze feature_category :devops_reports - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/analytics/usage_trends/counter_job_worker.rb b/app/workers/analytics/usage_trends/counter_job_worker.rb index bfb8a435939..b3a8f7dd3c2 100644 --- a/app/workers/analytics/usage_trends/counter_job_worker.rb +++ b/app/workers/analytics/usage_trends/counter_job_worker.rb @@ -12,7 +12,6 @@ module Analytics feature_category :devops_reports urgency :low - tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/approve_blocked_pending_approval_users_worker.rb b/app/workers/approve_blocked_pending_approval_users_worker.rb index fdf1bd99558..661ec87c1dd 100644 --- a/app/workers/approve_blocked_pending_approval_users_worker.rb +++ b/app/workers/approve_blocked_pending_approval_users_worker.rb @@ -10,7 +10,6 @@ class ApproveBlockedPendingApprovalUsersWorker idempotent! feature_category :users - tags :exclude_from_kubernetes def perform(current_user_id) current_user = User.find(current_user_id) diff --git a/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb b/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb index ef5dcc1cb99..48e3d0837c7 100644 --- a/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb +++ b/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb @@ -14,19 +14,12 @@ module AuthorizedProjectUpdate deduplicate :until_executing, including_scheduled: true def perform(user_id) - if Feature.enabled?(:user_refresh_from_replica_worker_uses_replica_db) - use_replica_if_available do - user = User.find_by_id(user_id) - - if user && project_authorizations_needs_refresh?(user) - enqueue_project_authorizations_refresh(user) - end - end - else + use_replica_if_available do user = User.find_by_id(user_id) - return unless user - user.refresh_authorized_projects(source: self.class.name) + if user && project_authorizations_needs_refresh?(user) + enqueue_project_authorizations_refresh(user) + end end end diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index ef58258d998..b771ab4d4e7 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -3,6 +3,8 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + MAX_LEASE_ATTEMPTS = 5 + data_consistency :always sidekiq_options retry: 3 @@ -30,10 +32,11 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker # lease_attempts - The number of times we will try to obtain an exclusive # lease on the class before giving up. See MR for more discussion. # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45298#note_434304956 - def perform(class_name, arguments = [], lease_attempts = 5) + def perform(class_name, arguments = [], lease_attempts = MAX_LEASE_ATTEMPTS) with_context(caller_id: class_name.to_s) do + retried = lease_attempts != MAX_LEASE_ATTEMPTS attempts_left = lease_attempts - 1 - should_perform, ttl = perform_and_ttl(class_name, attempts_left) + should_perform, ttl = perform_and_ttl(class_name, attempts_left, retried) break if should_perform.nil? @@ -50,13 +53,13 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker end end - def perform_and_ttl(class_name, attempts_left) + def perform_and_ttl(class_name, attempts_left, retried) # In test environments `perform_in` will run right away. This can then # lead to stack level errors in the above `#perform`. To work around this # we'll just perform the migration right away in the test environment. return [true, nil] if always_perform? - lease = lease_for(class_name) + lease = lease_for(class_name, retried) lease_obtained = !!lease.try_obtain healthy_db = healthy_database? perform = lease_obtained && healthy_db @@ -82,13 +85,17 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker [perform, lease.ttl] end - def lease_for(class_name) + def lease_for(class_name, retried) Gitlab::ExclusiveLease - .new(lease_key_for(class_name), timeout: self.class.minimum_interval) + .new(lease_key_for(class_name, retried), timeout: self.class.minimum_interval) end - def lease_key_for(class_name) - "#{self.class.name}:#{class_name}" + def lease_key_for(class_name, retried) + key = "#{self.class.name}:#{class_name}" + # We use a different exclusive lock key for retried jobs to allow them running concurrently with the scheduled jobs. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68763 for more information. + key += ":retried" if retried + key end def always_perform? diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb index 3f3d61a8df1..ce39ac946a9 100644 --- a/app/workers/build_success_worker.rb +++ b/app/workers/build_success_worker.rb @@ -21,8 +21,6 @@ class BuildSuccessWorker # rubocop:disable Scalability/IdempotentWorker private - ## - # TODO: This should be processed in DeploymentSuccessWorker once we started storing `action` value in `deployments` records def stop_environment(build) build.persisted_environment.fire_state_event(:stop) end diff --git a/app/workers/bulk_import_worker.rb b/app/workers/bulk_import_worker.rb index d7f0b752a34..fa255d064cc 100644 --- a/app/workers/bulk_import_worker.rb +++ b/app/workers/bulk_import_worker.rb @@ -6,7 +6,6 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker data_consistency :always feature_category :importers - tags :exclude_from_kubernetes sidekiq_options retry: false, dead: false @@ -25,9 +24,9 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker @bulk_import.start! if @bulk_import.created? created_entities.first(next_batch_size).each do |entity| - create_pipeline_tracker_for(entity) + entity.create_pipeline_trackers! - BulkImports::ExportRequestWorker.perform_async(entity.id) + BulkImports::ExportRequestWorker.perform_async(entity.id) if entity.group_entity? BulkImports::EntityWorker.perform_async(entity.id) entity.start! @@ -76,13 +75,4 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker def re_enqueue BulkImportWorker.perform_in(PERFORM_DELAY, @bulk_import.id) end - - def create_pipeline_tracker_for(entity) - BulkImports::Stage.pipelines.each do |stage, pipeline| - entity.trackers.create!( - stage: stage, - pipeline_name: pipeline - ) - end - end end diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb index cc52e349130..5c04cdc96a0 100644 --- a/app/workers/bulk_imports/entity_worker.rb +++ b/app/workers/bulk_imports/entity_worker.rb @@ -7,7 +7,6 @@ module BulkImports data_consistency :always feature_category :importers - tags :exclude_from_kubernetes sidekiq_options retry: false, dead: false diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb index 713c6c69213..760a309a381 100644 --- a/app/workers/bulk_imports/pipeline_worker.rb +++ b/app/workers/bulk_imports/pipeline_worker.rb @@ -9,7 +9,6 @@ module BulkImports NDJSON_PIPELINE_PERFORM_DELAY = 1.minute feature_category :importers - tags :exclude_from_kubernetes sidekiq_options retry: false, dead: false diff --git a/app/workers/bulk_imports/relation_export_worker.rb b/app/workers/bulk_imports/relation_export_worker.rb index 416dad5b3ae..9324b79cc75 100644 --- a/app/workers/bulk_imports/relation_export_worker.rb +++ b/app/workers/bulk_imports/relation_export_worker.rb @@ -10,7 +10,6 @@ module BulkImports idempotent! loggable_arguments 2, 3 feature_category :importers - tags :exclude_from_kubernetes sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION def perform(user_id, portable_id, portable_class, relation) diff --git a/app/workers/ci/delete_objects_worker.rb b/app/workers/ci/delete_objects_worker.rb index d31d248597b..cbcad3e8838 100644 --- a/app/workers/ci/delete_objects_worker.rb +++ b/app/workers/ci/delete_objects_worker.rb @@ -10,7 +10,6 @@ module Ci include LimitedCapacity::Worker feature_category :continuous_integration - tags :exclude_from_kubernetes idempotent! def perform_work(*args) diff --git a/app/workers/ci/drop_pipeline_worker.rb b/app/workers/ci/drop_pipeline_worker.rb index f3672dba3fe..edb97c3cac5 100644 --- a/app/workers/ci/drop_pipeline_worker.rb +++ b/app/workers/ci/drop_pipeline_worker.rb @@ -9,8 +9,6 @@ module Ci sidekiq_options retry: 3 include PipelineQueue - tags :exclude_from_kubernetes - idempotent! def perform(pipeline_id, failure_reason) diff --git a/app/workers/ci/external_pull_requests/create_pipeline_worker.rb b/app/workers/ci/external_pull_requests/create_pipeline_worker.rb new file mode 100644 index 00000000000..211ea1f2990 --- /dev/null +++ b/app/workers/ci/external_pull_requests/create_pipeline_worker.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Ci + module ExternalPullRequests + class CreatePipelineWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + data_consistency :always + queue_namespace :pipeline_creation + feature_category :pipeline_authoring + urgency :high + worker_resource_boundary :cpu + + def perform(project_id, user_id, external_pull_request_id) + user = User.find_by_id(user_id) + return unless user + + project = Project.find_by_id(project_id) + return unless project + + external_pull_request = project.external_pull_requests.find_by_id(external_pull_request_id) + return unless external_pull_request + + ::Ci::CreatePipelineService + .new(project, user, execute_params(external_pull_request)) + .execute(:external_pull_request_event, external_pull_request: external_pull_request) + end + + private + + def execute_params(pull_request) + { + ref: pull_request.source_ref, + source_sha: pull_request.source_sha, + target_sha: pull_request.target_sha + } + end + end + end +end diff --git a/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb b/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb index af042dc1e64..98bb259db0a 100644 --- a/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb +++ b/app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb @@ -10,7 +10,6 @@ module Ci include PipelineQueue urgency :low - tags :exclude_from_kubernetes idempotent! def perform(job_id) diff --git a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb index 06bc100c66a..bb0a81a0a17 100644 --- a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb +++ b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb @@ -11,7 +11,6 @@ module Ci queue_namespace :pipeline_background feature_category :code_testing - tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb b/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb index e4dc293353b..2af07cf6f93 100644 --- a/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb +++ b/app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb @@ -15,7 +15,6 @@ module Ci deduplicate :until_executed, including_scheduled: true idempotent! feature_category :continuous_integration - tags :exclude_from_kubernetes def perform service = ::Ci::PipelineArtifacts::DestroyAllExpiredService.new diff --git a/app/workers/ci/schedule_delete_objects_cron_worker.rb b/app/workers/ci/schedule_delete_objects_cron_worker.rb index 06bf83ae0a7..55b23bbab62 100644 --- a/app/workers/ci/schedule_delete_objects_cron_worker.rb +++ b/app/workers/ci/schedule_delete_objects_cron_worker.rb @@ -12,7 +12,6 @@ module Ci # rubocop:enable Scalability/CronWorkerContext feature_category :continuous_integration - tags :exclude_from_kubernetes idempotent! def perform(*args) diff --git a/app/workers/ci/test_failure_history_worker.rb b/app/workers/ci/test_failure_history_worker.rb index b67797edf0b..e79ca50c8ce 100644 --- a/app/workers/ci/test_failure_history_worker.rb +++ b/app/workers/ci/test_failure_history_worker.rb @@ -9,8 +9,6 @@ module Ci sidekiq_options retry: 3 include PipelineBackgroundQueue - tags :exclude_from_kubernetes - idempotent! def perform(pipeline_id) diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index 6cc6c30c5e9..3399a4f9b57 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -58,10 +58,7 @@ module ApplicationWorker Gitlab::SidekiqConfig::WorkerRouter.queue_name_from_worker_name(self) end - override :validate_worker_attributes! def validate_worker_attributes! - super - # Since the delayed data_consistency will use sidekiq built in retry mechanism, it is required that this mechanism # is not disabled. if retry_disabled? && get_data_consistency == :delayed @@ -81,6 +78,13 @@ module ApplicationWorker end end + override :data_consistency + def data_consistency(data_consistency, feature_flag: nil) + super + + validate_worker_attributes! + end + def perform_async(*args) # Worker execution for workers with data_consistency set to :delayed or :sticky # will be delayed to give replication enough time to complete diff --git a/app/workers/concerns/chaos_queue.rb b/app/workers/concerns/chaos_queue.rb index 2ccd55157c6..a9c557f0175 100644 --- a/app/workers/concerns/chaos_queue.rb +++ b/app/workers/concerns/chaos_queue.rb @@ -6,6 +6,5 @@ module ChaosQueue included do queue_namespace :chaos feature_category_not_owned! - tags :exclude_from_gitlab_com end end diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index 806fce38636..eebea30655c 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -92,17 +92,6 @@ module WorkerAttributes set_class_attribute(:data_consistency_feature_flag, feature_flag) if feature_flag set_class_attribute(:data_consistency, data_consistency) - - validate_worker_attributes! - end - - def validate_worker_attributes! - # Since the deduplication should always take into account the latest binary replication pointer into account, - # not the first one, the deduplication will not work with sticky or delayed. - # Follow up issue to improve this: https://gitlab.com/gitlab-org/gitlab/-/issues/325291 - if idempotent? && utilizes_load_balancing_capabilities? - raise ArgumentError, "Class can't be marked as idempotent if data_consistency is not set to :always" - end end # If data_consistency is not set to :always, worker will try to utilize load balancing capabilities and use the replica @@ -147,8 +136,6 @@ module WorkerAttributes def idempotent! set_class_attribute(:idempotent, true) - - validate_worker_attributes! end def idempotent? diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb index de9bb4d5a93..433ed5e0ea4 100644 --- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb +++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb @@ -12,7 +12,6 @@ module ContainerExpirationPolicies queue_namespace :container_repository feature_category :container_registry - tags :exclude_from_kubernetes urgency :low worker_resource_boundary :unknown idempotent! diff --git a/app/workers/database/batched_background_migration_worker.rb b/app/workers/database/batched_background_migration_worker.rb index 0750ff1acaf..fda539b372d 100644 --- a/app/workers/database/batched_background_migration_worker.rb +++ b/app/workers/database/batched_background_migration_worker.rb @@ -9,7 +9,6 @@ module Database include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :database - tags :exclude_from_kubernetes idempotent! LEASE_TIMEOUT_MULTIPLIER = 3 diff --git a/app/workers/database/partition_management_worker.rb b/app/workers/database/partition_management_worker.rb index a203c76558a..5a1f139dc29 100644 --- a/app/workers/database/partition_management_worker.rb +++ b/app/workers/database/partition_management_worker.rb @@ -12,7 +12,7 @@ module Database idempotent! def perform - Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions + Gitlab::Database::Partitioning.sync_partitions ensure Gitlab::Database::Partitioning::PartitionMonitoring.new.report_metrics end diff --git a/app/workers/deployments/drop_older_deployments_worker.rb b/app/workers/deployments/drop_older_deployments_worker.rb index 979f683cfb3..c464febd119 100644 --- a/app/workers/deployments/drop_older_deployments_worker.rb +++ b/app/workers/deployments/drop_older_deployments_worker.rb @@ -10,7 +10,6 @@ module Deployments queue_namespace :deployment feature_category :continuous_delivery - tags :exclude_from_kubernetes def perform(deployment_id) Deployments::OlderDeploymentsDropService.new(deployment_id).execute diff --git a/app/workers/deployments/finished_worker.rb b/app/workers/deployments/finished_worker.rb deleted file mode 100644 index 25121656408..00000000000 --- a/app/workers/deployments/finished_worker.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -# This worker is deprecated and will be removed in 14.0 -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/266381 -module Deployments - class FinishedWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - data_consistency :always - - sidekiq_options retry: 3 - - queue_namespace :deployment - feature_category :continuous_delivery - worker_resource_boundary :cpu - - def perform(deployment_id) - if (deploy = Deployment.find_by_id(deployment_id)) - LinkMergeRequestsService.new(deploy).execute - deploy.execute_hooks(Time.current) - end - end - end -end diff --git a/app/workers/deployments/forward_deployment_worker.rb b/app/workers/deployments/forward_deployment_worker.rb deleted file mode 100644 index 7f5eb13b88d..00000000000 --- a/app/workers/deployments/forward_deployment_worker.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -# This worker is deprecated and will be removed in 14.0 -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/266381 -module Deployments - class ForwardDeploymentWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - data_consistency :always - - sidekiq_options retry: 3 - - queue_namespace :deployment - feature_category :continuous_delivery - - def perform(deployment_id) - Deployments::OlderDeploymentsDropService.new(deployment_id).execute - end - end -end diff --git a/app/workers/deployments/hooks_worker.rb b/app/workers/deployments/hooks_worker.rb index d23a440ed36..31c57e5c001 100644 --- a/app/workers/deployments/hooks_worker.rb +++ b/app/workers/deployments/hooks_worker.rb @@ -4,7 +4,7 @@ module Deployments class HooksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - data_consistency :delayed, feature_flag: :load_balancing_for_deployments_hooks_worker + data_consistency :delayed queue_namespace :deployment feature_category :continuous_delivery diff --git a/app/workers/deployments/success_worker.rb b/app/workers/deployments/success_worker.rb deleted file mode 100644 index 401c2d7600c..00000000000 --- a/app/workers/deployments/success_worker.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -# This worker is deprecated and will be removed in 14.0 -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/266381 -module Deployments - class SuccessWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - data_consistency :always - - sidekiq_options retry: 3 - - queue_namespace :deployment - feature_category :continuous_delivery - worker_resource_boundary :cpu - - def perform(deployment_id) - Deployment.find_by_id(deployment_id).try do |deployment| - break unless deployment.success? - - Deployments::UpdateEnvironmentService.new(deployment).execute - end - end - end -end diff --git a/app/workers/design_management/copy_design_collection_worker.rb b/app/workers/design_management/copy_design_collection_worker.rb index a498eed173c..8b265979afa 100644 --- a/app/workers/design_management/copy_design_collection_worker.rb +++ b/app/workers/design_management/copy_design_collection_worker.rb @@ -9,7 +9,6 @@ module DesignManagement sidekiq_options retry: 3 feature_category :design_management - tags :exclude_from_kubernetes idempotent! urgency :low diff --git a/app/workers/destroy_pages_deployments_worker.rb b/app/workers/destroy_pages_deployments_worker.rb index 36424f7473e..7fa73648dd2 100644 --- a/app/workers/destroy_pages_deployments_worker.rb +++ b/app/workers/destroy_pages_deployments_worker.rb @@ -10,7 +10,6 @@ class DestroyPagesDeploymentsWorker loggable_arguments 0, 1 sidekiq_options retry: 3 feature_category :pages - tags :exclude_from_kubernetes def perform(project_id, last_deployment_id = nil) project = Project.find_by_id(project_id) diff --git a/app/workers/disallow_two_factor_for_group_worker.rb b/app/workers/disallow_two_factor_for_group_worker.rb index 4f5ef69a730..5b958f9f31f 100644 --- a/app/workers/disallow_two_factor_for_group_worker.rb +++ b/app/workers/disallow_two_factor_for_group_worker.rb @@ -9,7 +9,6 @@ class DisallowTwoFactorForGroupWorker include ExceptionBacktrace feature_category :subgroups - tags :exclude_from_kubernetes idempotent! def perform(group_id) diff --git a/app/workers/disallow_two_factor_for_subgroups_worker.rb b/app/workers/disallow_two_factor_for_subgroups_worker.rb index d3528b0674b..500c13deed2 100644 --- a/app/workers/disallow_two_factor_for_subgroups_worker.rb +++ b/app/workers/disallow_two_factor_for_subgroups_worker.rb @@ -11,7 +11,6 @@ class DisallowTwoFactorForSubgroupsWorker INTERVAL = 2.seconds.to_i feature_category :subgroups - tags :exclude_from_kubernetes idempotent! def perform(group_id) diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 1514897b2e4..51211834e06 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -11,7 +11,7 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker urgency :high weight 2 - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263 tags :needs_own_queue attr_accessor :raw diff --git a/app/workers/environments/auto_stop_worker.rb b/app/workers/environments/auto_stop_worker.rb new file mode 100644 index 00000000000..672a4f4121e --- /dev/null +++ b/app/workers/environments/auto_stop_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Environments + class AutoStopWorker + include ApplicationWorker + + data_consistency :always + idempotent! + feature_category :continuous_delivery + + def perform(environment_id, params = {}) + Environment.find_by_id(environment_id).try do |environment| + user = environment.stop_action&.user + environment.stop_with_action!(user) + end + end + end +end diff --git a/app/workers/environments/canary_ingress/update_worker.rb b/app/workers/environments/canary_ingress/update_worker.rb index 591c88cac96..02f24db1469 100644 --- a/app/workers/environments/canary_ingress/update_worker.rb +++ b/app/workers/environments/canary_ingress/update_worker.rb @@ -11,7 +11,6 @@ module Environments idempotent! worker_has_external_dependencies! feature_category :continuous_delivery - tags :exclude_from_kubernetes def perform(environment_id, params) Environment.find_by_id(environment_id).try do |environment| diff --git a/app/workers/experiments/record_conversion_event_worker.rb b/app/workers/experiments/record_conversion_event_worker.rb index 4c82c114d15..6487f030628 100644 --- a/app/workers/experiments/record_conversion_event_worker.rb +++ b/app/workers/experiments/record_conversion_event_worker.rb @@ -9,7 +9,6 @@ module Experiments sidekiq_options retry: 3 feature_category :users - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index cd5ca25f031..401fe1dc1e5 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -1,20 +1,24 @@ # frozen_string_literal: true -class ExpireJobCacheWorker +class ExpireJobCacheWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - data_consistency :always + data_consistency :delayed sidekiq_options retry: 3 include PipelineQueue queue_namespace :pipeline_cache urgency :high - idempotent! + # This worker should be idempotent, but we're switching to data_consistency + # :sticky and there is an ongoing incompatibility, so it needs to be disabled for + # now. The following line can be uncommented and this comment removed once + # https://gitlab.com/gitlab-org/gitlab/-/issues/325291 is resolved. + # idempotent! # rubocop: disable CodeReuse/ActiveRecord def perform(job_id) - job = CommitStatus.eager_load_pipeline.find_by(id: job_id) + job = CommitStatus.preload(:pipeline, :project).find_by(id: job_id) return unless job pipeline = job.pipeline diff --git a/app/workers/expire_pipeline_cache_worker.rb b/app/workers/expire_pipeline_cache_worker.rb index 64f73d1fba1..07e6939d1c7 100644 --- a/app/workers/expire_pipeline_cache_worker.rb +++ b/app/workers/expire_pipeline_cache_worker.rb @@ -19,7 +19,7 @@ class ExpirePipelineCacheWorker # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) - pipeline = Ci::Pipeline.eager_load_project.find_by(id: pipeline_id) + pipeline = Ci::Pipeline.find_by(id: pipeline_id) return unless pipeline Ci::ExpirePipelineCacheService.new.execute(pipeline) diff --git a/app/workers/flush_counter_increments_worker.rb b/app/workers/flush_counter_increments_worker.rb index bcb6a4c2bca..c4a3a5283cc 100644 --- a/app/workers/flush_counter_increments_worker.rb +++ b/app/workers/flush_counter_increments_worker.rb @@ -13,7 +13,6 @@ class FlushCounterIncrementsWorker sidekiq_options retry: 3 feature_category_not_owned! - tags :exclude_from_kubernetes urgency :low deduplicate :until_executing, including_scheduled: true diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb index cce179542c7..ab0cb81249b 100644 --- a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -5,7 +5,6 @@ module Gitlab class ImportPullRequestMergedByWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter - tags :exclude_from_kubernetes worker_resource_boundary :cpu def representation_class diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb index 8796d6392df..8d5c7b95b10 100644 --- a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -5,7 +5,6 @@ module Gitlab class ImportPullRequestReviewWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter - tags :exclude_from_kubernetes worker_resource_boundary :cpu def representation_class diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index c33836e20d1..5188bda03e2 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -12,17 +12,10 @@ module Gitlab include GithubImport::Queue include StageMethods - # The importers to run in this stage. Issues can't be imported earlier - # on as we also use these to enrich pull requests with assigned labels. - IMPORTERS = [ - Importer::IssuesImporter, - Importer::DiffNotesImporter - ].freeze - # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) - waiters = IMPORTERS.each_with_object({}) do |klass, hash| + waiters = importers(project).each_with_object({}) do |klass, hash| info(project.id, message: "starting importer", importer: klass.name) waiter = klass.new(project, client).execute hash[waiter.key] = waiter.jobs_remaining @@ -30,6 +23,25 @@ module Gitlab AdvanceStageWorker.perform_async(project.id, waiters, :notes) end + + # The importers to run in this stage. Issues can't be imported earlier + # on as we also use these to enrich pull requests with assigned labels. + def importers(project) + [ + Importer::IssuesImporter, + diff_notes_importer(project) + ] + end + + private + + def diff_notes_importer(project) + if project.group.present? && Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops, default_enabled: :yaml) + Importer::SingleEndpointDiffNotesImporter + else + Importer::DiffNotesImporter + end + end end end end diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 0160145ffe2..90a1337169f 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -15,17 +15,31 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) - info(project.id, message: "starting importer", importer: 'Importer::NotesImporter') - waiter = Importer::NotesImporter - .new(project, client) - .execute + waiters = importers(project).each_with_object({}) do |klass, hash| + info(project.id, message: "starting importer", importer: klass.name) + waiter = klass.new(project, client).execute + hash[waiter.key] = waiter.jobs_remaining + end AdvanceStageWorker.perform_async( project.id, - { waiter.key => waiter.jobs_remaining }, + waiters, :lfs_objects ) end + + def importers(project) + if project.group.present? && Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops, default_enabled: :yaml) + [ + Importer::SingleEndpointMergeRequestNotesImporter, + Importer::SingleEndpointIssueNotesImporter + ] + else + [ + Importer::NotesImporter + ] + end + end end end end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb index 7d83fe288da..8c2d652a689 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb @@ -12,8 +12,6 @@ module Gitlab include GithubImport::Queue include StageMethods - tags :exclude_from_kubernetes - # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb index ea3b89efd22..e10f1170618 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb @@ -12,8 +12,6 @@ module Gitlab include GithubImport::Queue include StageMethods - tags :exclude_from_kubernetes - # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) diff --git a/app/workers/gitlab_performance_bar_stats_worker.rb b/app/workers/gitlab_performance_bar_stats_worker.rb index 4e8bcb9af7b..6d637ad1586 100644 --- a/app/workers/gitlab_performance_bar_stats_worker.rb +++ b/app/workers/gitlab_performance_bar_stats_worker.rb @@ -15,7 +15,6 @@ class GitlabPerformanceBarStatsWorker STATS_KEY_EXPIRE = 30.minutes.to_i feature_category :metrics - tags :exclude_from_kubernetes idempotent! def perform(lease_uuid) diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb index f44c109f12d..92195d3fe16 100644 --- a/app/workers/group_destroy_worker.rb +++ b/app/workers/group_destroy_worker.rb @@ -9,7 +9,6 @@ class GroupDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ExceptionBacktrace feature_category :subgroups - tags :requires_disk_io, :exclude_from_kubernetes def perform(group_id, user_id) begin diff --git a/app/workers/hashed_storage/migrator_worker.rb b/app/workers/hashed_storage/migrator_worker.rb index 80e86fd7814..03019ae3131 100644 --- a/app/workers/hashed_storage/migrator_worker.rb +++ b/app/workers/hashed_storage/migrator_worker.rb @@ -11,9 +11,8 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - # Gitlab::HashedStorage::Migrator#migration_pending? depends on the - # queue size of this worker. - tags :exclude_from_gitlab_com, :needs_own_queue + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 + tags :needs_own_queue # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index edddea55356..bcc80cc2a70 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -11,9 +11,8 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - # Gitlab::HashedStorage::Migrator#migration_pending? depends on the - # queue size of this worker. - tags :exclude_from_gitlab_com, :needs_own_queue + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 + tags :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index c5841dbbb28..07a7ab63718 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -11,9 +11,8 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the - # queue size of this worker. - tags :exclude_from_gitlab_com, :needs_own_queue + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 + tags :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/rollbacker_worker.rb b/app/workers/hashed_storage/rollbacker_worker.rb index 90e48f0e37a..d6a16b4d083 100644 --- a/app/workers/hashed_storage/rollbacker_worker.rb +++ b/app/workers/hashed_storage/rollbacker_worker.rb @@ -11,9 +11,8 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the - # queue size of this worker. - tags :exclude_from_gitlab_com, :needs_own_queue + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 + tags :needs_own_queue # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/incident_management/add_severity_system_note_worker.rb b/app/workers/incident_management/add_severity_system_note_worker.rb index 31da7b0bcfe..3a4667bea0a 100644 --- a/app/workers/incident_management/add_severity_system_note_worker.rb +++ b/app/workers/incident_management/add_severity_system_note_worker.rb @@ -11,7 +11,6 @@ module IncidentManagement queue_namespace :incident_management feature_category :incident_management - tags :exclude_from_kubernetes def perform(incident_id, user_id) return if incident_id.blank? || user_id.blank? diff --git a/app/workers/issue_rebalancing_worker.rb b/app/workers/issue_rebalancing_worker.rb index 13e02c37bdb..01984197aae 100644 --- a/app/workers/issue_rebalancing_worker.rb +++ b/app/workers/issue_rebalancing_worker.rb @@ -10,7 +10,6 @@ class IssueRebalancingWorker idempotent! urgency :low feature_category :issue_tracking - tags :exclude_from_kubernetes deduplicate :until_executed, including_scheduled: true def perform(ignore = nil, project_id = nil, root_namespace_id = nil) @@ -33,12 +32,8 @@ class IssueRebalancingWorker return end - # Temporary disable rebalancing for performance reasons - # For more information check https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4321 - return if projects_to_rebalance.take&.root_namespace&.issue_repositioning_disabled? # rubocop:disable CodeReuse/ActiveRecord - - IssueRebalancingService.new(projects_to_rebalance).execute - rescue IssueRebalancingService::TooManyIssues => e + Issues::RelativePositionRebalancingService.new(projects_to_rebalance).execute + rescue Issues::RelativePositionRebalancingService::TooManyConcurrentRebalances => e Gitlab::ErrorTracking.log_exception(e, root_namespace_id: root_namespace_id, project_id: project_id) end diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb index 379f087bc1b..1a9974af55f 100644 --- a/app/workers/jira_connect/sync_builds_worker.rb +++ b/app/workers/jira_connect/sync_builds_worker.rb @@ -8,7 +8,6 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations data_consistency :delayed - tags :exclude_from_kubernetes urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_deployments_worker.rb b/app/workers/jira_connect/sync_deployments_worker.rb index 3138230ced5..f4286752359 100644 --- a/app/workers/jira_connect/sync_deployments_worker.rb +++ b/app/workers/jira_connect/sync_deployments_worker.rb @@ -8,7 +8,6 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations data_consistency :delayed - tags :exclude_from_kubernetes urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_feature_flags_worker.rb b/app/workers/jira_connect/sync_feature_flags_worker.rb index 4de27c1b551..507d5c29f38 100644 --- a/app/workers/jira_connect/sync_feature_flags_worker.rb +++ b/app/workers/jira_connect/sync_feature_flags_worker.rb @@ -8,7 +8,6 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations data_consistency :delayed - tags :exclude_from_kubernetes urgency :low worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_project_worker.rb b/app/workers/jira_connect/sync_project_worker.rb index a83444e58e7..b0ebaf30e99 100644 --- a/app/workers/jira_connect/sync_project_worker.rb +++ b/app/workers/jira_connect/sync_project_worker.rb @@ -8,7 +8,6 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations data_consistency :delayed - tags :exclude_from_kubernetes urgency :low worker_has_external_dependencies! diff --git a/app/workers/member_invitation_reminder_emails_worker.rb b/app/workers/member_invitation_reminder_emails_worker.rb index fa6787b9063..a7614db30f6 100644 --- a/app/workers/member_invitation_reminder_emails_worker.rb +++ b/app/workers/member_invitation_reminder_emails_worker.rb @@ -8,7 +8,6 @@ class MemberInvitationReminderEmailsWorker # rubocop:disable Scalability/Idempot include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :subgroups - tags :exclude_from_kubernetes urgency :low def perform diff --git a/app/workers/merge_request_cleanup_refs_worker.rb b/app/workers/merge_request_cleanup_refs_worker.rb index c57c6fbc28d..27bd5774b8d 100644 --- a/app/workers/merge_request_cleanup_refs_worker.rb +++ b/app/workers/merge_request_cleanup_refs_worker.rb @@ -10,7 +10,6 @@ class MergeRequestCleanupRefsWorker sidekiq_options retry: 3 feature_category :code_review - tags :exclude_from_kubernetes idempotent! # Hard-coded to 4 for now. Will be configurable later on via application settings. diff --git a/app/workers/metrics/dashboard/sync_dashboards_worker.rb b/app/workers/metrics/dashboard/sync_dashboards_worker.rb index 645c03428a2..fe8694582c4 100644 --- a/app/workers/metrics/dashboard/sync_dashboards_worker.rb +++ b/app/workers/metrics/dashboard/sync_dashboards_worker.rb @@ -10,7 +10,6 @@ module Metrics sidekiq_options retry: 3 feature_category :metrics - tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/namespaces/in_product_marketing_emails_worker.rb b/app/workers/namespaces/in_product_marketing_emails_worker.rb index 035fa453f59..49e65d59e83 100644 --- a/app/workers/namespaces/in_product_marketing_emails_worker.rb +++ b/app/workers/namespaces/in_product_marketing_emails_worker.rb @@ -9,7 +9,6 @@ module Namespaces include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :subgroups - tags :exclude_from_kubernetes urgency :low def perform diff --git a/app/workers/namespaces/onboarding_issue_created_worker.rb b/app/workers/namespaces/onboarding_issue_created_worker.rb index 3cff741ecbf..81d105ab19c 100644 --- a/app/workers/namespaces/onboarding_issue_created_worker.rb +++ b/app/workers/namespaces/onboarding_issue_created_worker.rb @@ -9,7 +9,6 @@ module Namespaces sidekiq_options retry: 3 feature_category :issue_tracking - tags :exclude_from_kubernetes urgency :low deduplicate :until_executing diff --git a/app/workers/namespaces/onboarding_pipeline_created_worker.rb b/app/workers/namespaces/onboarding_pipeline_created_worker.rb index 2c77fab8114..f9a6b734586 100644 --- a/app/workers/namespaces/onboarding_pipeline_created_worker.rb +++ b/app/workers/namespaces/onboarding_pipeline_created_worker.rb @@ -9,7 +9,6 @@ module Namespaces sidekiq_options retry: 3 feature_category :subgroups - tags :exclude_from_kubernetes urgency :low deduplicate :until_executing diff --git a/app/workers/namespaces/onboarding_progress_worker.rb b/app/workers/namespaces/onboarding_progress_worker.rb index 43d13618091..b77db1aec5e 100644 --- a/app/workers/namespaces/onboarding_progress_worker.rb +++ b/app/workers/namespaces/onboarding_progress_worker.rb @@ -10,7 +10,6 @@ module Namespaces feature_category :product_analytics worker_resource_boundary :cpu - tags :exclude_from_kubernetes urgency :low deduplicate :until_executed diff --git a/app/workers/namespaces/onboarding_user_added_worker.rb b/app/workers/namespaces/onboarding_user_added_worker.rb index 4d4d9c03d3e..6a189e81b95 100644 --- a/app/workers/namespaces/onboarding_user_added_worker.rb +++ b/app/workers/namespaces/onboarding_user_added_worker.rb @@ -9,7 +9,6 @@ module Namespaces sidekiq_options retry: 3 feature_category :users - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/packages/composer/cache_cleanup_worker.rb b/app/workers/packages/composer/cache_cleanup_worker.rb index fee886bc750..19babf63967 100644 --- a/app/workers/packages/composer/cache_cleanup_worker.rb +++ b/app/workers/packages/composer/cache_cleanup_worker.rb @@ -10,7 +10,6 @@ module Packages include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :package_registry - tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/packages/composer/cache_update_worker.rb b/app/workers/packages/composer/cache_update_worker.rb index f146a0a83cc..874993a1325 100644 --- a/app/workers/packages/composer/cache_update_worker.rb +++ b/app/workers/packages/composer/cache_update_worker.rb @@ -10,7 +10,6 @@ module Packages sidekiq_options retry: 3 feature_category :package_registry - tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/packages/debian/process_changes_worker.rb b/app/workers/packages/debian/process_changes_worker.rb index 3ddeb858429..0a716c61203 100644 --- a/app/workers/packages/debian/process_changes_worker.rb +++ b/app/workers/packages/debian/process_changes_worker.rb @@ -13,7 +13,6 @@ module Packages queue_namespace :package_repositories feature_category :package_registry - tags :exclude_from_kubernetes def perform(package_file_id, user_id) @package_file_id = package_file_id @@ -22,12 +21,7 @@ module Packages return unless package_file && user ::Packages::Debian::ProcessChangesService.new(package_file, user).execute - rescue ArgumentError, - Packages::Debian::ExtractChangesMetadataService::ExtractionError, - Packages::Debian::ExtractDebMetadataService::CommandFailedError, - Packages::Debian::ExtractMetadataService::ExtractionError, - Packages::Debian::ParseDebian822Service::InvalidDebian822Error, - ActiveRecord::RecordNotFound => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e, package_file_id: @package_file_id, user_id: @user_id) package_file.destroy! end diff --git a/app/workers/packages/go/sync_packages_worker.rb b/app/workers/packages/go/sync_packages_worker.rb index 182c9bfec5d..dbf48cea1ae 100644 --- a/app/workers/packages/go/sync_packages_worker.rb +++ b/app/workers/packages/go/sync_packages_worker.rb @@ -12,7 +12,6 @@ module Packages queue_namespace :package_repositories feature_category :package_registry - tags :exclude_from_kubernetes deduplicate :until_executing idempotent! diff --git a/app/workers/packages/helm/extraction_worker.rb b/app/workers/packages/helm/extraction_worker.rb index 1010a0833b1..0ba2d149f77 100644 --- a/app/workers/packages/helm/extraction_worker.rb +++ b/app/workers/packages/helm/extraction_worker.rb @@ -20,9 +20,7 @@ module Packages ::Packages::Helm::ProcessFileService.new(channel, package_file).execute - rescue ::Packages::Helm::ExtractFileMetadataService::ExtractionError, - ::Packages::Helm::ProcessFileService::ExtractionError, - ::ActiveModel::ValidationError => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e, project_id: package_file.project_id) package_file.package.update_column(:status, :error) end diff --git a/app/workers/packages/maven/metadata/sync_worker.rb b/app/workers/packages/maven/metadata/sync_worker.rb index ab18c70e95e..b18b950e1c8 100644 --- a/app/workers/packages/maven/metadata/sync_worker.rb +++ b/app/workers/packages/maven/metadata/sync_worker.rb @@ -13,7 +13,6 @@ module Packages queue_namespace :package_repositories feature_category :package_registry - tags :exclude_from_kubernetes deduplicate :until_executing idempotent! diff --git a/app/workers/packages/rubygems/extraction_worker.rb b/app/workers/packages/rubygems/extraction_worker.rb index 520305981cf..dbaf9bc35a9 100644 --- a/app/workers/packages/rubygems/extraction_worker.rb +++ b/app/workers/packages/rubygems/extraction_worker.rb @@ -11,7 +11,6 @@ module Packages queue_namespace :package_repositories feature_category :package_registry - tags :exclude_from_kubernetes deduplicate :until_executing def perform(package_file_id) diff --git a/app/workers/pages_domain_ssl_renewal_worker.rb b/app/workers/pages_domain_ssl_renewal_worker.rb index d4c68f66699..d6b40318fd4 100644 --- a/app/workers/pages_domain_ssl_renewal_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_worker.rb @@ -8,7 +8,6 @@ class PagesDomainSslRenewalWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 feature_category :pages - tags :requires_disk_io, :exclude_from_kubernetes def perform(domain_id) domain = PagesDomain.find_by_id(domain_id) diff --git a/app/workers/pages_domain_verification_worker.rb b/app/workers/pages_domain_verification_worker.rb index f9504a7c1d2..59de00d40d5 100644 --- a/app/workers/pages_domain_verification_worker.rb +++ b/app/workers/pages_domain_verification_worker.rb @@ -8,7 +8,6 @@ class PagesDomainVerificationWorker # rubocop:disable Scalability/IdempotentWork sidekiq_options retry: 3 feature_category :pages - tags :requires_disk_io, :exclude_from_kubernetes # rubocop: disable CodeReuse/ActiveRecord def perform(domain_id) diff --git a/app/workers/pages_remove_worker.rb b/app/workers/pages_remove_worker.rb index 69a8344b5aa..4de99b8654d 100644 --- a/app/workers/pages_remove_worker.rb +++ b/app/workers/pages_remove_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# TODO: remove this worker https://gitlab.com/gitlab-org/gitlab/-/issues/320775 +# TODO: remove this worker https://gitlab.com/gitlab-org/gitlab/-/issues/340641 class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -8,13 +8,9 @@ class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 feature_category :pages - tags :exclude_from_kubernetes loggable_arguments 0 def perform(project_id) - project = Project.find_by_id(project_id) - return unless project - - project.legacy_remove_pages + # no-op end end diff --git a/app/workers/pages_transfer_worker.rb b/app/workers/pages_transfer_worker.rb index c2190a352dd..404c79b9e89 100644 --- a/app/workers/pages_transfer_worker.rb +++ b/app/workers/pages_transfer_worker.rb @@ -10,7 +10,6 @@ class PagesTransferWorker # rubocop:disable Scalability/IdempotentWorker TransferFailedError = Class.new(StandardError) feature_category :pages - tags :exclude_from_kubernetes loggable_arguments 0, 1 def perform(method, args) diff --git a/app/workers/pages_update_configuration_worker.rb b/app/workers/pages_update_configuration_worker.rb index 8a37b70a0b7..3dfd82ed517 100644 --- a/app/workers/pages_update_configuration_worker.rb +++ b/app/workers/pages_update_configuration_worker.rb @@ -9,7 +9,6 @@ class PagesUpdateConfigurationWorker idempotent! feature_category :pages - tags :exclude_from_kubernetes def self.perform_async(*args) return unless ::Settings.pages.local_store.enabled diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index d0c21cf74e1..5e951ab2c3a 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -8,7 +8,6 @@ class PagesWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 feature_category :pages loggable_arguments 0, 1 - tags :requires_disk_io, :exclude_from_kubernetes worker_resource_boundary :cpu def perform(action, *arg) diff --git a/app/workers/personal_access_tokens/expired_notification_worker.rb b/app/workers/personal_access_tokens/expired_notification_worker.rb index 8deacf457b2..2d0ea3d3aa4 100644 --- a/app/workers/personal_access_tokens/expired_notification_worker.rb +++ b/app/workers/personal_access_tokens/expired_notification_worker.rb @@ -9,7 +9,6 @@ module PersonalAccessTokens include CronjobQueue feature_category :authentication_and_authorization - tags :exclude_from_kubernetes def perform(*args) notification_service = NotificationService.new diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 4a49e18eb9b..7d0322361b8 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -1,8 +1,10 @@ # frozen_string_literal: true -class PostReceive # rubocop:disable Scalability/IdempotentWorker +class PostReceive include ApplicationWorker + idempotent! + deduplicate :none data_consistency :always sidekiq_options retry: 3 diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb index 149f8290b54..45d0ebd2b65 100644 --- a/app/workers/project_destroy_worker.rb +++ b/app/workers/project_destroy_worker.rb @@ -9,7 +9,6 @@ class ProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ExceptionBacktrace feature_category :source_code_management - tags :requires_disk_io, :exclude_from_kubernetes def perform(project_id, user_id, params) project = Project.find(project_id) diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb index 0d67a8ac30e..cf236f8b660 100644 --- a/app/workers/projects/git_garbage_collect_worker.rb +++ b/app/workers/projects/git_garbage_collect_worker.rb @@ -5,8 +5,6 @@ module Projects extend ::Gitlab::Utils::Override include GitGarbageCollectMethods - tags :exclude_from_kubernetes - private override :find_resource diff --git a/app/workers/projects/post_creation_worker.rb b/app/workers/projects/post_creation_worker.rb index 99438e4e4b2..3a39bd17ce3 100644 --- a/app/workers/projects/post_creation_worker.rb +++ b/app/workers/projects/post_creation_worker.rb @@ -9,7 +9,6 @@ module Projects sidekiq_options retry: 3 feature_category :source_code_management - tags :exclude_from_kubernetes idempotent! def perform(project_id) diff --git a/app/workers/propagate_integration_group_worker.rb b/app/workers/propagate_integration_group_worker.rb index 443ff1f2abe..ed08e90f38d 100644 --- a/app/workers/propagate_integration_group_worker.rb +++ b/app/workers/propagate_integration_group_worker.rb @@ -6,7 +6,6 @@ class PropagateIntegrationGroupWorker data_consistency :always sidekiq_options retry: 3 feature_category :integrations - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/propagate_integration_inherit_descendant_worker.rb b/app/workers/propagate_integration_inherit_descendant_worker.rb index 24573591409..8b3ecc1f057 100644 --- a/app/workers/propagate_integration_inherit_descendant_worker.rb +++ b/app/workers/propagate_integration_inherit_descendant_worker.rb @@ -6,7 +6,6 @@ class PropagateIntegrationInheritDescendantWorker data_consistency :always sidekiq_options retry: 3 feature_category :integrations - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/propagate_integration_inherit_worker.rb b/app/workers/propagate_integration_inherit_worker.rb index 24a8778b928..f0a53f8cb07 100644 --- a/app/workers/propagate_integration_inherit_worker.rb +++ b/app/workers/propagate_integration_inherit_worker.rb @@ -6,7 +6,6 @@ class PropagateIntegrationInheritWorker data_consistency :always sidekiq_options retry: 3 feature_category :integrations - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/propagate_integration_project_worker.rb b/app/workers/propagate_integration_project_worker.rb index dba8a270007..bc55b7ff504 100644 --- a/app/workers/propagate_integration_project_worker.rb +++ b/app/workers/propagate_integration_project_worker.rb @@ -6,7 +6,6 @@ class PropagateIntegrationProjectWorker data_consistency :always sidekiq_options retry: 3 feature_category :integrations - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/purge_dependency_proxy_cache_worker.rb b/app/workers/purge_dependency_proxy_cache_worker.rb index 8ab4e77bc78..db43e4adf20 100644 --- a/app/workers/purge_dependency_proxy_cache_worker.rb +++ b/app/workers/purge_dependency_proxy_cache_worker.rb @@ -27,6 +27,6 @@ class PurgeDependencyProxyCacheWorker def valid? return unless @group - can?(@current_user, :admin_group, @group) && @group.dependency_proxy_feature_available? + can?(@current_user, :admin_group, @group) end end diff --git a/app/workers/releases/create_evidence_worker.rb b/app/workers/releases/create_evidence_worker.rb index 5aed543500f..628b0e0b806 100644 --- a/app/workers/releases/create_evidence_worker.rb +++ b/app/workers/releases/create_evidence_worker.rb @@ -9,7 +9,6 @@ module Releases sidekiq_options retry: 3 feature_category :release_evidence - tags :exclude_from_kubernetes # pipeline_id is optional for backward compatibility with existing jobs # caller should always try to provide the pipeline and pass nil only diff --git a/app/workers/releases/manage_evidence_worker.rb b/app/workers/releases/manage_evidence_worker.rb index f316aa6eefd..94d0342a65a 100644 --- a/app/workers/releases/manage_evidence_worker.rb +++ b/app/workers/releases/manage_evidence_worker.rb @@ -9,7 +9,6 @@ module Releases include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :release_evidence - tags :exclude_from_kubernetes def perform releases = Release.without_evidence.released_within_2hrs diff --git a/app/workers/remove_unaccepted_member_invites_worker.rb b/app/workers/remove_unaccepted_member_invites_worker.rb index 7833ec30c3c..7fe45b26094 100644 --- a/app/workers/remove_unaccepted_member_invites_worker.rb +++ b/app/workers/remove_unaccepted_member_invites_worker.rb @@ -8,7 +8,6 @@ class RemoveUnacceptedMemberInvitesWorker # rubocop:disable Scalability/Idempote include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :authentication_and_authorization - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/schedule_merge_request_cleanup_refs_worker.rb b/app/workers/schedule_merge_request_cleanup_refs_worker.rb index 46a6e0ef01f..58cd8f7ade3 100644 --- a/app/workers/schedule_merge_request_cleanup_refs_worker.rb +++ b/app/workers/schedule_merge_request_cleanup_refs_worker.rb @@ -8,7 +8,6 @@ class ScheduleMergeRequestCleanupRefsWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :code_review - tags :exclude_from_kubernetes idempotent! def perform diff --git a/app/workers/service_desk_email_receiver_worker.rb b/app/workers/service_desk_email_receiver_worker.rb index f546fce3e8a..c8ab8891856 100644 --- a/app/workers/service_desk_email_receiver_worker.rb +++ b/app/workers/service_desk_email_receiver_worker.rb @@ -6,9 +6,10 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Sca data_consistency :always feature_category :service_desk + urgency :high sidekiq_options retry: 3 - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263 tags :needs_own_queue def should_perform? diff --git a/app/workers/ssh_keys/expired_notification_worker.rb b/app/workers/ssh_keys/expired_notification_worker.rb index 6afeecdd1b5..d8553b5a9a2 100644 --- a/app/workers/ssh_keys/expired_notification_worker.rb +++ b/app/workers/ssh_keys/expired_notification_worker.rb @@ -9,7 +9,6 @@ module SshKeys include CronjobQueue feature_category :compliance_management - tags :exclude_from_kubernetes idempotent! BATCH_SIZE = 500 diff --git a/app/workers/ssh_keys/expiring_soon_notification_worker.rb b/app/workers/ssh_keys/expiring_soon_notification_worker.rb index ef256621e07..a89520867ed 100644 --- a/app/workers/ssh_keys/expiring_soon_notification_worker.rb +++ b/app/workers/ssh_keys/expiring_soon_notification_worker.rb @@ -9,7 +9,6 @@ module SshKeys include CronjobQueue feature_category :compliance_management - tags :exclude_from_kubernetes idempotent! def perform diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 5723380a3f3..a2b2686c8d5 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -3,59 +3,29 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - data_consistency :always - + # rubocop:disable Scalability/CronWorkerContext + # This is an instance-wide cleanup query, so there's no meaningful + # scope to consider this in the context of. include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + data_consistency :always feature_category :continuous_integration worker_resource_boundary :cpu EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease' - BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour - BUILD_PENDING_OUTDATED_TIMEOUT = 1.day - BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour - BUILD_PENDING_STUCK_TIMEOUT = 1.hour - BUILD_LOOKBACK = 5.days - def perform return unless try_obtain_lease - Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds" - - drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure) - - drop( - Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: BUILD_PENDING_OUTDATED_TIMEOUT.ago), - failure_reason: :stuck_or_timeout_failure - ) - - drop(scheduled_timed_out_builds, failure_reason: :stale_schedule) - - drop_stuck( - Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: BUILD_PENDING_STUCK_TIMEOUT.ago), - failure_reason: :stuck_or_timeout_failure - ) + Ci::StuckBuilds::DropService.new.execute remove_lease end private - def scheduled_timed_out_builds - Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord - 'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?', - BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago - ) - end - - def running_timed_out_builds - Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord - 'ci_builds.updated_at < ?', - BUILD_RUNNING_OUTDATED_TIMEOUT.ago - ) - end - def try_obtain_lease @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain end @@ -63,55 +33,4 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker def remove_lease Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) end - - def drop(builds, failure_reason:) - fetch(builds) do |build| - drop_build :outdated, build, failure_reason - end - end - - def drop_stuck(builds, failure_reason:) - fetch(builds) do |build| - break unless build.stuck? - - drop_build :stuck, build, failure_reason - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def fetch(builds) - loop do - jobs = builds.includes(:tags, :runner, project: [:namespace, :route]) - .limit(100) - .to_a - - break if jobs.empty? - - jobs.each do |job| - with_context(project: job.project) { yield(job) } - end - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def drop_build(type, build, reason) - Gitlab::AppLogger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{build.status}, failure_reason: #{reason})" - Gitlab::OptimisticLocking.retry_lock(build, 3, name: 'stuck_ci_jobs_worker_drop_build') do |b| - b.drop(reason) - end - rescue StandardError => ex - build.doom! - - track_exception_for_build(ex, build) - end - - def track_exception_for_build(ex, build) - Gitlab::ErrorTracking.track_exception(ex, - build_id: build.id, - build_name: build.name, - build_stage: build.stage, - pipeline_id: build.pipeline_id, - project_id: build.project_id - ) - end end diff --git a/app/workers/todos_destroyer/destroyed_designs_worker.rb b/app/workers/todos_destroyer/destroyed_designs_worker.rb new file mode 100644 index 00000000000..e01c1a109d7 --- /dev/null +++ b/app/workers/todos_destroyer/destroyed_designs_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module TodosDestroyer + class DestroyedDesignsWorker + include ApplicationWorker + + data_consistency :always + + sidekiq_options retry: 3 + include TodosDestroyerQueue + + idempotent! + + def perform(design_ids) + ::Todos::Destroy::DesignService.new(design_ids).execute + end + end +end diff --git a/app/workers/todos_destroyer/destroyed_issuable_worker.rb b/app/workers/todos_destroyer/destroyed_issuable_worker.rb index ff4f5e15472..f93c1389534 100644 --- a/app/workers/todos_destroyer/destroyed_issuable_worker.rb +++ b/app/workers/todos_destroyer/destroyed_issuable_worker.rb @@ -9,8 +9,6 @@ module TodosDestroyer sidekiq_options retry: 3 include TodosDestroyerQueue - tags :exclude_from_kubernetes - idempotent! def perform(target_id, target_type) diff --git a/app/workers/user_status_cleanup/batch_worker.rb b/app/workers/user_status_cleanup/batch_worker.rb index b6ca6548572..f3d73b2e6e9 100644 --- a/app/workers/user_status_cleanup/batch_worker.rb +++ b/app/workers/user_status_cleanup/batch_worker.rb @@ -12,7 +12,6 @@ module UserStatusCleanup # rubocop:enable Scalability/CronWorkerContext feature_category :users - tags :exclude_from_kubernetes idempotent! diff --git a/app/workers/users/deactivate_dormant_users_worker.rb b/app/workers/users/deactivate_dormant_users_worker.rb index bcb13483379..d7ea20e4b62 100644 --- a/app/workers/users/deactivate_dormant_users_worker.rb +++ b/app/workers/users/deactivate_dormant_users_worker.rb @@ -9,7 +9,6 @@ module Users include CronjobQueue feature_category :utilization - tags :exclude_from_kubernetes NUMBER_OF_BATCHES = 50 BATCH_SIZE = 200 diff --git a/app/workers/web_hooks/destroy_worker.rb b/app/workers/web_hooks/destroy_worker.rb index b92fe86bafb..f457cd11e54 100644 --- a/app/workers/web_hooks/destroy_worker.rb +++ b/app/workers/web_hooks/destroy_worker.rb @@ -7,7 +7,6 @@ module WebHooks data_consistency :always sidekiq_options retry: 3 feature_category :integrations - tags :exclude_from_kubernetes urgency :low idempotent! diff --git a/app/workers/web_hooks/log_execution_worker.rb b/app/workers/web_hooks/log_execution_worker.rb index 50d91182c80..280d987fa77 100644 --- a/app/workers/web_hooks/log_execution_worker.rb +++ b/app/workers/web_hooks/log_execution_worker.rb @@ -7,6 +7,8 @@ module WebHooks data_consistency :always feature_category :integrations urgency :low + sidekiq_options retry: 3 + loggable_arguments 0, 2, 3 idempotent! diff --git a/app/workers/wikis/git_garbage_collect_worker.rb b/app/workers/wikis/git_garbage_collect_worker.rb index f34d3be51d2..1b455c50618 100644 --- a/app/workers/wikis/git_garbage_collect_worker.rb +++ b/app/workers/wikis/git_garbage_collect_worker.rb @@ -5,8 +5,6 @@ module Wikis extend ::Gitlab::Utils::Override include GitGarbageCollectMethods - tags :exclude_from_kubernetes - private override :find_resource |