diff options
Diffstat (limited to 'app/workers')
194 files changed, 1020 insertions, 592 deletions
diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb index a7cc4fb0d11..c84ac60d777 100644 --- a/app/workers/admin_email_worker.rb +++ b/app/workers/admin_email_worker.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true -class AdminEmailWorker +class AdminEmailWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue # rubocop:enable Scalability/CronWorkerContext - feature_category_not_owned! + feature_category :source_code_management def perform send_repository_check_mail if Gitlab::CurrentSettings.repository_checks_enabled diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f6daab73689..28fab10d931 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -6,1080 +6,1295 @@ - :name: auto_devops:auto_devops_disable :feature_category: :auto_devops :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: auto_merge:auto_merge_process :feature_category: :continuous_delivery :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: chaos:chaos_cpu_spin :feature_category: :chaos_engineering :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: chaos:chaos_db_spin :feature_category: :chaos_engineering :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: chaos:chaos_kill :feature_category: :chaos_engineering :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: chaos:chaos_leak_mem :feature_category: :chaos_engineering :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: chaos:chaos_sleep :feature_category: :chaos_engineering :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: container_repository:cleanup_container_repository :feature_category: :container_registry :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: container_repository:delete_container_repository :feature_category: :container_registry :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:admin_email - :feature_category: :not_owned + :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:ci_archive_traces_cron :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:container_expiration_policy :feature_category: :container_registry :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:environments_auto_stop_cron :feature_category: :continuous_delivery :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:expire_build_artifacts :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:gitlab_usage_ping - :feature_category: :not_owned + :feature_category: :collection :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:import_export_project_cleanup :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:issue_due_scheduler :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:namespaces_prune_aggregation_schedules :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: cronjob:pages_domain_removal_cron :feature_category: :pages :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: cronjob:pages_domain_ssl_renewal_cron :feature_category: :pages :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:pages_domain_verification_cron :feature_category: :pages :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:personal_access_tokens_expiring :feature_category: :authentication_and_authorization :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:pipeline_schedule :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: cronjob:prune_old_events - :feature_category: :not_owned + :feature_category: :users :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:prune_web_hook_logs :feature_category: :integrations :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:remove_expired_group_links :feature_category: :authentication_and_authorization :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:remove_expired_members :feature_category: :authentication_and_authorization :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: cronjob:remove_unreferenced_lfs_objects :feature_category: :git_lfs :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:repository_archive_cache :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:repository_check_dispatch :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:requests_profiles :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:schedule_migrate_external_diffs :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:stuck_ci_jobs :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: +- :name: cronjob:stuck_export_jobs + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: - :name: cronjob:stuck_import_jobs :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: cronjob:stuck_merge_jobs :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: cronjob:trending_projects :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: deployment:deployments_finished :feature_category: :continuous_delivery :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: deployment:deployments_forward_deployment :feature_category: :continuous_delivery :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 3 + :idempotent: - :name: deployment:deployments_success :feature_category: :continuous_delivery :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: gcp_cluster:cluster_configure :feature_category: :kubernetes_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_configure_istio :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_install_app :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_patch_app :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_project_configure :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_provision :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_upgrade_app :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_wait_for_app_installation :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: gcp_cluster:cluster_wait_for_ingress_ip_address :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:clusters_applications_activate_service :feature_category: :kubernetes_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:clusters_applications_deactivate_service :feature_category: :kubernetes_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:clusters_applications_uninstall :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:clusters_applications_wait_for_uninstall_app :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: gcp_cluster:clusters_cleanup_app :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:clusters_cleanup_project_namespace :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:clusters_cleanup_service_account :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gcp_cluster:wait_for_cluster_creation :feature_category: :kubernetes_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_import_diff_note :feature_category: :importers :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_import_issue :feature_category: :importers :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_import_lfs_object :feature_category: :importers :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_import_note :feature_category: :importers :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_import_pull_request :feature_category: :importers :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_refresh_import_jid :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_stage_finish_import :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_stage_import_base_data :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_stage_import_issues_and_diff_notes :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_stage_import_lfs_objects :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_stage_import_notes :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_stage_import_pull_requests :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_importer:github_import_stage_import_repository :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: hashed_storage:hashed_storage_migrator :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: hashed_storage:hashed_storage_project_migrate :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: hashed_storage:hashed_storage_project_rollback :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: hashed_storage:hashed_storage_rollbacker :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: incident_management:incident_management_process_alert :feature_category: :incident_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: mail_scheduler:mail_scheduler_issue_due :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: mail_scheduler:mail_scheduler_notification_service :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 2 + :idempotent: - :name: notifications:new_release :feature_category: :release_orchestration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: object_pool:object_pool_create :feature_category: :gitaly :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: object_pool:object_pool_destroy :feature_category: :gitaly :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: object_pool:object_pool_join :feature_category: :gitaly :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: object_pool:object_pool_schedule_join :feature_category: :gitaly :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: object_storage:object_storage_background_move :feature_category: :not_owned :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: object_storage:object_storage_migrate_uploads :feature_category: :not_owned :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: pipeline_background:archive_trace :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: pipeline_background:ci_build_trace_chunk_flush :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: pipeline_cache:expire_job_cache :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 3 + :idempotent: true - :name: pipeline_cache:expire_pipeline_cache :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: pipeline_creation:create_pipeline :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 4 + :idempotent: - :name: pipeline_creation:run_pipeline_schedule :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 4 + :idempotent: - :name: pipeline_default:build_coverage :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 3 + :idempotent: - :name: pipeline_default:build_trace_sections :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 3 + :idempotent: - :name: pipeline_default:ci_create_cross_project_pipeline :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: pipeline_default:ci_pipeline_bridge_status :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: pipeline_default:pipeline_metrics :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 3 + :idempotent: - :name: pipeline_default:pipeline_notification :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high + :resource_boundary: :cpu + :weight: 3 + :idempotent: +- :name: pipeline_default:pipeline_update_ci_ref_status + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :high :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: pipeline_hooks:build_hooks :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: pipeline_hooks:pipeline_hooks :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 2 + :idempotent: - :name: pipeline_processing:build_finished :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 5 + :idempotent: - :name: pipeline_processing:build_queue :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 5 + :idempotent: - :name: pipeline_processing:build_success :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: pipeline_processing:ci_build_prepare :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: pipeline_processing:ci_build_schedule :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 5 + :idempotent: - :name: pipeline_processing:ci_resource_groups_assign_resource_from_resource_group :feature_category: :continuous_delivery :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: pipeline_processing:pipeline_process :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: pipeline_processing:pipeline_success :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: pipeline_processing:pipeline_update :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: pipeline_processing:stage_update :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: pipeline_processing:update_head_pipeline_for_merge_request :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 5 + :idempotent: - :name: repository_check:repository_check_batch :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: repository_check:repository_check_clear :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: repository_check:repository_check_single_repository :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: todos_destroyer:todos_destroyer_confidential_issue :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: todos_destroyer:todos_destroyer_entity_leave :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: todos_destroyer:todos_destroyer_group_private :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: todos_destroyer:todos_destroyer_private_features :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: todos_destroyer:todos_destroyer_project_private :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: update_namespace_statistics:namespaces_root_statistics :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: update_namespace_statistics:namespaces_schedule_aggregation :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: +- :name: authorized_keys + :feature_category: :source_code_management + :has_external_dependencies: + :urgency: :high + :resource_boundary: :unknown + :weight: 2 + :idempotent: true - :name: authorized_projects :feature_category: :authentication_and_authorization :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 2 + :idempotent: true - :name: background_migration :feature_category: :not_owned :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: chat_notification :feature_category: :chatops - :has_external_dependencies: - :latency_sensitive: true + :has_external_dependencies: true + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: create_commit_signature :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: create_evidence - :feature_category: :release_governance + :feature_category: :release_evidence :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: create_note_diff_file :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: default :feature_category: :has_external_dependencies: - :latency_sensitive: + :urgency: :resource_boundary: :weight: 1 + :idempotent: - :name: delete_diff_files :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: delete_merged_branches :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: delete_stored_files :feature_category: :not_owned :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: delete_user :feature_category: :authentication_and_authorization :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: detect_repository_languages :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: email_receiver :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: emails_on_push :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 2 + :idempotent: - :name: error_tracking_issue_link :feature_category: :error_tracking :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: expire_build_instance_artifacts :feature_category: :continuous_integration :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: file_hook :feature_category: :integrations :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: git_garbage_collect :feature_category: :gitaly :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: github_import_advance_stage :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: gitlab_shell :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: group_destroy :feature_category: :subgroups :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: group_export :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: group_import :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: import_issues_csv :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :cpu :weight: 2 + :idempotent: - :name: invalid_gpg_signature_update :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: irker :feature_category: :integrations :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: mailers :feature_category: :has_external_dependencies: - :latency_sensitive: + :urgency: :resource_boundary: :weight: 2 + :idempotent: - :name: merge :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 5 + :idempotent: - :name: merge_request_mergeability_check :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: migrate_external_diffs :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: namespaceless_project_destroy :feature_category: :authentication_and_authorization :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: new_issue :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 2 + :idempotent: - :name: new_merge_request :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 2 + :idempotent: - :name: new_note :feature_category: :issue_tracking :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 2 + :idempotent: - :name: pages :feature_category: :pages :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: pages_domain_ssl_renewal :feature_category: :pages :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: pages_domain_verification :feature_category: :pages :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: phabricator_import_import_tasks :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: post_receive :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 5 + :idempotent: - :name: process_commit :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 3 + :idempotent: - :name: project_cache :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: project_daily_statistics :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: project_destroy :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: project_export :feature_category: :importers :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :memory :weight: 1 + :idempotent: - :name: project_service :feature_category: :integrations :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: +- :name: project_update_repository_storage + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: - :name: propagate_service_template :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: reactive_caching :feature_category: :not_owned :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 1 + :idempotent: - :name: rebase :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: remote_mirror_notification :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: repository_cleanup :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: repository_fork :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: repository_import :feature_category: :importers :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: repository_remove_remote :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: repository_update_remote_mirror :feature_category: :source_code_management :has_external_dependencies: true - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: self_monitoring_project_create :feature_category: :metrics :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: self_monitoring_project_delete :feature_category: :metrics :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 2 + :idempotent: - :name: system_hook_push :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: update_external_pull_requests :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 3 + :idempotent: - :name: update_merge_requests :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: true + :urgency: :high :resource_boundary: :cpu :weight: 3 + :idempotent: - :name: update_project_statistics :feature_category: :source_code_management :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: upload_checksum :feature_category: :geo_replication :has_external_dependencies: - :latency_sensitive: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: - :name: web_hook :feature_category: :integrations :has_external_dependencies: true - :latency_sensitive: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: +- :name: x509_certificate_revoke + :feature_category: :source_code_management + :has_external_dependencies: + :urgency: :low :resource_boundary: :unknown :weight: 1 + :idempotent: true diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb index 66f9b8d9e80..3ddb5686bf2 100644 --- a/app/workers/archive_trace_worker.rb +++ b/app/workers/archive_trace_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ArchiveTraceWorker +class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineBackgroundQueue diff --git a/app/workers/authorized_keys_worker.rb b/app/workers/authorized_keys_worker.rb new file mode 100644 index 00000000000..b2333033e56 --- /dev/null +++ b/app/workers/authorized_keys_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class AuthorizedKeysWorker + include ApplicationWorker + + PERMITTED_ACTIONS = [:add_key, :remove_key].freeze + + feature_category :source_code_management + urgency :high + weight 2 + idempotent! + + def perform(action, *args) + return unless Gitlab::CurrentSettings.authorized_keys_enabled? + + case action + when :add_key + authorized_keys.add_key(*args) + when :remove_key + authorized_keys.remove_key(*args) + end + end + + private + + def authorized_keys + @authorized_keys ||= Gitlab::AuthorizedKeys.new + end +end diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index 1ab2fd6023f..a35e0320553 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -5,9 +5,11 @@ class AuthorizedProjectsWorker prepend WaitableWorker feature_category :authentication_and_authorization - latency_sensitive_worker! + urgency :high weight 2 + idempotent! + # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore the # visibility of prepended modules. See https://github.com/rspec/rspec-mocks/issues/1231 # for more details. diff --git a/app/workers/auto_devops/disable_worker.rb b/app/workers/auto_devops/disable_worker.rb index 73ddc591505..bae08cf9e18 100644 --- a/app/workers/auto_devops/disable_worker.rb +++ b/app/workers/auto_devops/disable_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module AutoDevops - class DisableWorker + class DisableWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include AutoDevopsQueue diff --git a/app/workers/auto_merge_process_worker.rb b/app/workers/auto_merge_process_worker.rb index 1681fac3363..2599c76c900 100644 --- a/app/workers/auto_merge_process_worker.rb +++ b/app/workers/auto_merge_process_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class AutoMergeProcessWorker +class AutoMergeProcessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :auto_merge diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index 20e2cdd7f96..231c2bcd83b 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class BackgroundMigrationWorker +class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category_not_owned! @@ -22,17 +22,19 @@ class BackgroundMigrationWorker # class_name - The class name of the background migration to run. # arguments - The arguments to pass to the migration class. def perform(class_name, arguments = []) - should_perform, ttl = perform_and_ttl(class_name) + with_context(caller_id: class_name.to_s) do + should_perform, ttl = perform_and_ttl(class_name) - if should_perform - Gitlab::BackgroundMigration.perform(class_name, arguments) - else - # If the lease could not be obtained this means either another process is - # running a migration of this class or we ran one recently. In this case - # we'll reschedule the job in such a way that it is picked up again around - # the time the lease expires. - self.class - .perform_in(ttl || self.class.minimum_interval, class_name, arguments) + if should_perform + Gitlab::BackgroundMigration.perform(class_name, arguments) + else + # If the lease could not be obtained this means either another process is + # running a migration of this class or we ran one recently. In this case + # we'll reschedule the job in such a way that it is picked up again around + # the time the lease expires. + self.class + .perform_in(ttl || self.class.minimum_interval, class_name, arguments) + end end end diff --git a/app/workers/build_coverage_worker.rb b/app/workers/build_coverage_worker.rb index 912c53e11f8..7d893024abc 100644 --- a/app/workers/build_coverage_worker.rb +++ b/app/workers/build_coverage_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class BuildCoverageWorker +class BuildCoverageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index 77ce0923307..b6ef9ab4710 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class BuildFinishedWorker +class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb index fa55769e486..9693d3eb57f 100644 --- a/app/workers/build_hooks_worker.rb +++ b/app/workers/build_hooks_worker.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -class BuildHooksWorker +class BuildHooksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_hooks feature_category :continuous_integration - latency_sensitive_worker! + urgency :high # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/build_queue_worker.rb b/app/workers/build_queue_worker.rb index 6f75f403e6e..b71afbbeb8f 100644 --- a/app/workers/build_queue_worker.rb +++ b/app/workers/build_queue_worker.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -class BuildQueueWorker +class BuildQueueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing feature_category :continuous_integration - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb index b7dbd367fee..e4a2dd500cc 100644 --- a/app/workers/build_success_worker.rb +++ b/app/workers/build_success_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class BuildSuccessWorker +class BuildSuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing - latency_sensitive_worker! + urgency :high # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb index 0641130fd64..c25f77974e9 100644 --- a/app/workers/build_trace_sections_worker.rb +++ b/app/workers/build_trace_sections_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class BuildTraceSectionsWorker +class BuildTraceSectionsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue diff --git a/app/workers/chaos/cpu_spin_worker.rb b/app/workers/chaos/cpu_spin_worker.rb index 43a32c3274f..0b565e0d49c 100644 --- a/app/workers/chaos/cpu_spin_worker.rb +++ b/app/workers/chaos/cpu_spin_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Chaos - class CpuSpinWorker + class CpuSpinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ChaosQueue diff --git a/app/workers/chaos/db_spin_worker.rb b/app/workers/chaos/db_spin_worker.rb index 217ddabbcb6..099660d440c 100644 --- a/app/workers/chaos/db_spin_worker.rb +++ b/app/workers/chaos/db_spin_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Chaos - class DbSpinWorker + class DbSpinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ChaosQueue diff --git a/app/workers/chaos/kill_worker.rb b/app/workers/chaos/kill_worker.rb index 80f04db1be4..3dedd47a1f9 100644 --- a/app/workers/chaos/kill_worker.rb +++ b/app/workers/chaos/kill_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Chaos - class KillWorker + class KillWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ChaosQueue diff --git a/app/workers/chaos/leak_mem_worker.rb b/app/workers/chaos/leak_mem_worker.rb index 0caa99e0de9..b77d1a20541 100644 --- a/app/workers/chaos/leak_mem_worker.rb +++ b/app/workers/chaos/leak_mem_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Chaos - class LeakMemWorker + class LeakMemWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ChaosQueue diff --git a/app/workers/chaos/sleep_worker.rb b/app/workers/chaos/sleep_worker.rb index 7c724c4cb4e..6887258e961 100644 --- a/app/workers/chaos/sleep_worker.rb +++ b/app/workers/chaos/sleep_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Chaos - class SleepWorker + class SleepWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ChaosQueue diff --git a/app/workers/chat_notification_worker.rb b/app/workers/chat_notification_worker.rb index f23c787559f..5fab437f49f 100644 --- a/app/workers/chat_notification_worker.rb +++ b/app/workers/chat_notification_worker.rb @@ -1,19 +1,15 @@ # frozen_string_literal: true -class ChatNotificationWorker +class ChatNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker TimeoutExceeded = Class.new(StandardError) sidekiq_options retry: false feature_category :chatops - latency_sensitive_worker! + urgency :low # Can't be high as it has external dependencies weight 2 - - # TODO: break this into multiple jobs - # as the `responder` uses external dependencies - # See https://gitlab.com/gitlab-com/gl-infra/scalability/issues/34 - # worker_has_external_dependencies! + worker_has_external_dependencies! RESCHEDULE_INTERVAL = 2.seconds RESCHEDULE_TIMEOUT = 5.minutes diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb index c73c7ba2dd8..0171c1d482d 100644 --- a/app/workers/ci/archive_traces_cron_worker.rb +++ b/app/workers/ci/archive_traces_cron_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class ArchiveTracesCronWorker + class ArchiveTracesCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/ci/build_prepare_worker.rb b/app/workers/ci/build_prepare_worker.rb index 20208c18d03..7f640633070 100644 --- a/app/workers/ci/build_prepare_worker.rb +++ b/app/workers/ci/build_prepare_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class BuildPrepareWorker + class BuildPrepareWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index e34f16f46c2..9231b40978d 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class BuildScheduleWorker + class BuildScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue diff --git a/app/workers/ci/build_trace_chunk_flush_worker.rb b/app/workers/ci/build_trace_chunk_flush_worker.rb index 23a11c28f9b..fe59ba896a4 100644 --- a/app/workers/ci/build_trace_chunk_flush_worker.rb +++ b/app/workers/ci/build_trace_chunk_flush_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class BuildTraceChunkFlushWorker + class BuildTraceChunkFlushWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineBackgroundQueue diff --git a/app/workers/ci/create_cross_project_pipeline_worker.rb b/app/workers/ci/create_cross_project_pipeline_worker.rb index 91e9317713e..713d0092b32 100644 --- a/app/workers/ci/create_cross_project_pipeline_worker.rb +++ b/app/workers/ci/create_cross_project_pipeline_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class CreateCrossProjectPipelineWorker + class CreateCrossProjectPipelineWorker # rubocop:disable Scalability/IdempotentWorker include ::ApplicationWorker include ::PipelineQueue diff --git a/app/workers/ci/pipeline_bridge_status_worker.rb b/app/workers/ci/pipeline_bridge_status_worker.rb index f196573deaa..3f92f4561e0 100644 --- a/app/workers/ci/pipeline_bridge_status_worker.rb +++ b/app/workers/ci/pipeline_bridge_status_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true module Ci - class PipelineBridgeStatusWorker + class PipelineBridgeStatusWorker # rubocop:disable Scalability/IdempotentWorker include ::ApplicationWorker include ::PipelineQueue - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu def perform(pipeline_id) diff --git a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb index 62233d19516..8063e34a1b8 100644 --- a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb +++ b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb @@ -2,7 +2,7 @@ module Ci module ResourceGroups - class AssignResourceFromResourceGroupWorker + class AssignResourceFromResourceGroupWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb index 83397a1dda2..c3fac453e73 100644 --- a/app/workers/cleanup_container_repository_worker.rb +++ b/app/workers/cleanup_container_repository_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class CleanupContainerRepositoryWorker +class CleanupContainerRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :container_repository diff --git a/app/workers/cluster_configure_istio_worker.rb b/app/workers/cluster_configure_istio_worker.rb index dfdd408f286..ec6bdfbd6b6 100644 --- a/app/workers/cluster_configure_istio_worker.rb +++ b/app/workers/cluster_configure_istio_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterConfigureIstioWorker +class ClusterConfigureIstioWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue diff --git a/app/workers/cluster_configure_worker.rb b/app/workers/cluster_configure_worker.rb index e7a4797e68e..f9364ab7144 100644 --- a/app/workers/cluster_configure_worker.rb +++ b/app/workers/cluster_configure_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterConfigureWorker +class ClusterConfigureWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb index 0e075b295dd..002932a0fa5 100644 --- a/app/workers/cluster_install_app_worker.rb +++ b/app/workers/cluster_install_app_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterInstallAppWorker +class ClusterInstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_patch_app_worker.rb b/app/workers/cluster_patch_app_worker.rb index 3f95a764567..f75004aa3e5 100644 --- a/app/workers/cluster_patch_app_worker.rb +++ b/app/workers/cluster_patch_app_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterPatchAppWorker +class ClusterPatchAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_project_configure_worker.rb b/app/workers/cluster_project_configure_worker.rb index 614029c2b5c..b68df01dc7a 100644 --- a/app/workers/cluster_project_configure_worker.rb +++ b/app/workers/cluster_project_configure_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterProjectConfigureWorker +class ClusterProjectConfigureWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb index c34284319dd..cb750f3021e 100644 --- a/app/workers/cluster_provision_worker.rb +++ b/app/workers/cluster_provision_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterProvisionWorker +class ClusterProvisionWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue diff --git a/app/workers/cluster_upgrade_app_worker.rb b/app/workers/cluster_upgrade_app_worker.rb index cd06f0a2224..99f48415f08 100644 --- a/app/workers/cluster_upgrade_app_worker.rb +++ b/app/workers/cluster_upgrade_app_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterUpgradeAppWorker +class ClusterUpgradeAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb index 7155dc6f835..e098c3b86b5 100644 --- a/app/workers/cluster_wait_for_app_installation_worker.rb +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterWaitForAppInstallationWorker +class ClusterWaitForAppInstallationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb index 14b1651cc72..c7336ee515d 100644 --- a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb +++ b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterWaitForIngressIpAddressWorker +class ClusterWaitForIngressIpAddressWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/clusters/applications/activate_service_worker.rb b/app/workers/clusters/applications/activate_service_worker.rb index 4f285d55162..abd7f8eaea4 100644 --- a/app/workers/clusters/applications/activate_service_worker.rb +++ b/app/workers/clusters/applications/activate_service_worker.rb @@ -2,7 +2,7 @@ module Clusters module Applications - class ActivateServiceWorker + class ActivateServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue diff --git a/app/workers/clusters/applications/deactivate_service_worker.rb b/app/workers/clusters/applications/deactivate_service_worker.rb index 2c560cc998c..fecbb6dde45 100644 --- a/app/workers/clusters/applications/deactivate_service_worker.rb +++ b/app/workers/clusters/applications/deactivate_service_worker.rb @@ -2,7 +2,7 @@ module Clusters module Applications - class DeactivateServiceWorker + class DeactivateServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue diff --git a/app/workers/clusters/applications/uninstall_worker.rb b/app/workers/clusters/applications/uninstall_worker.rb index 6180998c8d9..977a25e8442 100644 --- a/app/workers/clusters/applications/uninstall_worker.rb +++ b/app/workers/clusters/applications/uninstall_worker.rb @@ -2,7 +2,7 @@ module Clusters module Applications - class UninstallWorker + class UninstallWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb index 7907aa8dfff..a486cfa90b7 100644 --- a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb +++ b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb @@ -2,7 +2,7 @@ module Clusters module Applications - class WaitForUninstallAppWorker + class WaitForUninstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/clusters/cleanup/app_worker.rb b/app/workers/clusters/cleanup/app_worker.rb index 8b2fddd3164..1d01cec174b 100644 --- a/app/workers/clusters/cleanup/app_worker.rb +++ b/app/workers/clusters/cleanup/app_worker.rb @@ -2,7 +2,7 @@ module Clusters module Cleanup - class AppWorker + class AppWorker # rubocop:disable Scalability/IdempotentWorker include ClusterCleanupMethods def perform(cluster_id, execution_count = 0) diff --git a/app/workers/clusters/cleanup/project_namespace_worker.rb b/app/workers/clusters/cleanup/project_namespace_worker.rb index 8a7fbf0fde7..a7a951f2937 100644 --- a/app/workers/clusters/cleanup/project_namespace_worker.rb +++ b/app/workers/clusters/cleanup/project_namespace_worker.rb @@ -2,7 +2,7 @@ module Clusters module Cleanup - class ProjectNamespaceWorker + class ProjectNamespaceWorker # rubocop:disable Scalability/IdempotentWorker include ClusterCleanupMethods def perform(cluster_id, execution_count = 0) diff --git a/app/workers/clusters/cleanup/service_account_worker.rb b/app/workers/clusters/cleanup/service_account_worker.rb index 95de56d8ebe..a829d68fb20 100644 --- a/app/workers/clusters/cleanup/service_account_worker.rb +++ b/app/workers/clusters/cleanup/service_account_worker.rb @@ -2,7 +2,7 @@ module Clusters module Cleanup - class ServiceAccountWorker + class ServiceAccountWorker # rubocop:disable Scalability/IdempotentWorker include ClusterCleanupMethods def perform(cluster_id) diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index 733156ab758..c0062780688 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -13,6 +13,17 @@ module ApplicationWorker included do set_queue + + def structured_payload(payload = {}) + context = Labkit::Context.current.to_h.merge( + 'class' => self.class, + 'job_status' => 'running', + 'queue' => self.class.queue, + 'jid' => jid + ) + + payload.stringify_keys.merge(context) + end end class_methods do diff --git a/app/workers/concerns/gitlab/github_import/notify_upon_death.rb b/app/workers/concerns/gitlab/github_import/notify_upon_death.rb deleted file mode 100644 index 3d7120665b6..00000000000 --- a/app/workers/concerns/gitlab/github_import/notify_upon_death.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - # NotifyUponDeath can be included into a GitHub worker class if it should - # notify any JobWaiter instances upon being moved to the Sidekiq dead queue. - # - # Note that this will only notify the waiter upon graceful termination, a - # SIGKILL will still result in the waiter _not_ being notified. - # - # Workers including this module must have jobs passed where the last - # argument is the key to notify, as a String. - module NotifyUponDeath - extend ActiveSupport::Concern - - included do - # If a job is being exhausted we still want to notify the - # AdvanceStageWorker. This prevents the entire import from getting stuck - # just because 1 job threw too many errors. - sidekiq_retries_exhausted do |job| - args = job['args'] - jid = job['jid'] - - if args.length == 3 && (key = args.last) && key.is_a?(String) - JobWaiter.notify(key, jid) - end - end - end - end - end -end diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index bd0b566658e..63c1ba8e699 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -11,7 +11,7 @@ module Gitlab include ApplicationWorker include GithubImport::Queue include ReschedulingMethods - include NotifyUponDeath + include Gitlab::NotifyUponDeath feature_category :importers worker_has_external_dependencies! diff --git a/app/workers/concerns/gitlab/notify_upon_death.rb b/app/workers/concerns/gitlab/notify_upon_death.rb new file mode 100644 index 00000000000..66dc6270637 --- /dev/null +++ b/app/workers/concerns/gitlab/notify_upon_death.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + # NotifyUponDeath can be included into a worker class if it should + # notify any JobWaiter instances upon being moved to the Sidekiq dead queue. + # + # Note that this will only notify the waiter upon graceful termination, a + # SIGKILL will still result in the waiter _not_ being notified. + # + # Workers including this module must have jobs passed where the last + # argument is the key to notify, as a String. + module NotifyUponDeath + extend ActiveSupport::Concern + + included do + # If a job is being exhausted we still want to notify the + # Gitlab::Import::AdvanceStageWorker. This prevents the entire import from getting stuck + # just because 1 job threw too many errors. + sidekiq_retries_exhausted do |job| + args = job['args'] + jid = job['jid'] + + if args.length == 3 && (key = args.last) && key.is_a?(String) + JobWaiter.notify(key, jid) + end + end + end + end +end diff --git a/app/workers/concerns/project_export_options.rb b/app/workers/concerns/project_export_options.rb new file mode 100644 index 00000000000..e9318c1ba43 --- /dev/null +++ b/app/workers/concerns/project_export_options.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module ProjectExportOptions + extend ActiveSupport::Concern + + EXPORT_RETRY_COUNT = 3 + + included do + sidekiq_options retry: EXPORT_RETRY_COUNT, status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION + + # We mark the project export as failed once we have exhausted all retries + sidekiq_retries_exhausted do |job| + project = Project.find(job['args'][1]) + # rubocop: disable CodeReuse/ActiveRecord + job = project.export_jobs.find_by(jid: job["jid"]) + # rubocop: enable CodeReuse/ActiveRecord + + if job&.fail_op + Sidekiq.logger.info "Job #{job['jid']} for project #{project.id} has been set to failed state" + else + Sidekiq.logger.error "Failed to set Job #{job['jid']} for project #{project.id} to failed state" + end + end + end +end diff --git a/app/workers/concerns/waitable_worker.rb b/app/workers/concerns/waitable_worker.rb index 17946bbc5ca..f995aced542 100644 --- a/app/workers/concerns/waitable_worker.rb +++ b/app/workers/concerns/waitable_worker.rb @@ -9,7 +9,7 @@ module WaitableWorker # Short-circuit: it's more efficient to do small numbers of jobs inline return bulk_perform_inline(args_list) if args_list.size <= 3 - waiter = Gitlab::JobWaiter.new(args_list.size) + waiter = Gitlab::JobWaiter.new(args_list.size, worker_label: self.to_s) # Point all the bulk jobs at the same JobWaiter. Converts, [[1], [2], [3]] # into [[1, "key"], [2, "key"], [3, "key"]] diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index babdb46bb85..b60179531af 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -4,9 +4,12 @@ module WorkerAttributes extend ActiveSupport::Concern # Resource boundaries that workers can declare through the - # `worker_resource_boundary` attribute + # `resource_boundary` attribute VALID_RESOURCE_BOUNDARIES = [:memory, :cpu, :unknown].freeze + # Urgencies that workers can declare through the `urgencies` attribute + VALID_URGENCIES = [:high, :low, :throttled].freeze + NAMESPACE_WEIGHTS = { auto_devops: 2, auto_merge: 3, @@ -47,21 +50,22 @@ module WorkerAttributes get_worker_attribute(:feature_category) == :not_owned end - # This should be set for jobs that need to be run immediately, or, if - # they are delayed, risk creating inconsistencies in the application - # that could being perceived by the user as incorrect behavior - # (ie, a bug) - # See doc/development/sidekiq_style_guide.md#Latency-Sensitive-Jobs + # This should be set to :high for jobs that need to be run + # immediately, or, if they are delayed, risk creating + # inconsistencies in the application that could being perceived by + # the user as incorrect behavior (ie, a bug) + # + # See + # doc/development/sidekiq_style_guide.md#urgency # for details - def latency_sensitive_worker! - worker_attributes[:latency_sensitive] = true + def urgency(urgency) + raise "Invalid urgency: #{urgency}" unless VALID_URGENCIES.include?(urgency) + + worker_attributes[:urgency] = urgency end - # Returns a truthy value if the worker is latency sensitive. - # See doc/development/sidekiq_style_guide.md#Latency-Sensitive-Jobs - # for details - def latency_sensitive_worker? - worker_attributes[:latency_sensitive] + def get_urgency + worker_attributes[:urgency] || :low end # Set this attribute on a job when it will call to services outside of the @@ -89,6 +93,14 @@ module WorkerAttributes worker_attributes[:resource_boundary] || :unknown end + def idempotent! + worker_attributes[:idempotent] = true + end + + def idempotent? + worker_attributes[:idempotent] + end + def weight(value) worker_attributes[:weight] = value end diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb index e07a6546e2d..e1544be5aed 100644 --- a/app/workers/container_expiration_policy_worker.rb +++ b/app/workers/container_expiration_policy_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ContainerExpirationPolicyWorker +class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue diff --git a/app/workers/create_commit_signature_worker.rb b/app/workers/create_commit_signature_worker.rb index 027fea3e402..3da21c56eff 100644 --- a/app/workers/create_commit_signature_worker.rb +++ b/app/workers/create_commit_signature_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class CreateCommitSignatureWorker +class CreateCommitSignatureWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/create_evidence_worker.rb b/app/workers/create_evidence_worker.rb index e6fbf59d702..c2faba84cfc 100644 --- a/app/workers/create_evidence_worker.rb +++ b/app/workers/create_evidence_worker.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -class CreateEvidenceWorker +class CreateEvidenceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - feature_category :release_governance + feature_category :release_evidence weight 2 def perform(release_id) diff --git a/app/workers/create_note_diff_file_worker.rb b/app/workers/create_note_diff_file_worker.rb index ca200bd17b4..8a1709f04e1 100644 --- a/app/workers/create_note_diff_file_worker.rb +++ b/app/workers/create_note_diff_file_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class CreateNoteDiffFileWorker +class CreateNoteDiffFileWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/create_pipeline_worker.rb b/app/workers/create_pipeline_worker.rb index a75cc643038..54698518e4f 100644 --- a/app/workers/create_pipeline_worker.rb +++ b/app/workers/create_pipeline_worker.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -class CreatePipelineWorker +class CreatePipelineWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_creation feature_category :continuous_integration - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu def perform(project_id, user_id, ref, source, params = {}) diff --git a/app/workers/delete_container_repository_worker.rb b/app/workers/delete_container_repository_worker.rb index e70b4fb0a58..dbfc273a5ce 100644 --- a/app/workers/delete_container_repository_worker.rb +++ b/app/workers/delete_container_repository_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class DeleteContainerRepositoryWorker +class DeleteContainerRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExclusiveLeaseGuard diff --git a/app/workers/delete_diff_files_worker.rb b/app/workers/delete_diff_files_worker.rb index e0c1724f1f7..a6759a9d7c4 100644 --- a/app/workers/delete_diff_files_worker.rb +++ b/app/workers/delete_diff_files_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class DeleteDiffFilesWorker +class DeleteDiffFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/delete_merged_branches_worker.rb b/app/workers/delete_merged_branches_worker.rb index f3d86233c1b..ab3d42e5384 100644 --- a/app/workers/delete_merged_branches_worker.rb +++ b/app/workers/delete_merged_branches_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class DeleteMergedBranchesWorker +class DeleteMergedBranchesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/delete_stored_files_worker.rb b/app/workers/delete_stored_files_worker.rb index e1e2f66f573..463f26fdd5a 100644 --- a/app/workers/delete_stored_files_worker.rb +++ b/app/workers/delete_stored_files_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class DeleteStoredFilesWorker +class DeleteStoredFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category_not_owned! diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb index 0e49e787d8a..d3b87c133d3 100644 --- a/app/workers/delete_user_worker.rb +++ b/app/workers/delete_user_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class DeleteUserWorker +class DeleteUserWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :authentication_and_authorization diff --git a/app/workers/deployments/finished_worker.rb b/app/workers/deployments/finished_worker.rb index 6196b032f63..0be420af718 100644 --- a/app/workers/deployments/finished_worker.rb +++ b/app/workers/deployments/finished_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Deployments - class FinishedWorker + class FinishedWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :deployment diff --git a/app/workers/deployments/forward_deployment_worker.rb b/app/workers/deployments/forward_deployment_worker.rb index a25b8ca0478..a6f246dbbbd 100644 --- a/app/workers/deployments/forward_deployment_worker.rb +++ b/app/workers/deployments/forward_deployment_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Deployments - class ForwardDeploymentWorker + class ForwardDeploymentWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :deployment diff --git a/app/workers/deployments/success_worker.rb b/app/workers/deployments/success_worker.rb index 4a29f1aef52..17f790d2f6f 100644 --- a/app/workers/deployments/success_worker.rb +++ b/app/workers/deployments/success_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Deployments - class SuccessWorker + class SuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :deployment diff --git a/app/workers/detect_repository_languages_worker.rb b/app/workers/detect_repository_languages_worker.rb index 954d0f9336b..ef66287a692 100644 --- a/app/workers/detect_repository_languages_worker.rb +++ b/app/workers/detect_repository_languages_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class DetectRepositoryLanguagesWorker +class DetectRepositoryLanguagesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace include ExclusiveLeaseGuard diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index c2b1e642604..fcb88982c0b 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -class EmailReceiverWorker +class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :issue_tracking - latency_sensitive_worker! + urgency :high weight 2 def perform(raw) diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index be66e2b1188..cc114acf7e9 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -class EmailsOnPushWorker +class EmailsOnPushWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker attr_reader :email, :skip_premailer feature_category :source_code_management - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu weight 2 diff --git a/app/workers/environments/auto_stop_cron_worker.rb b/app/workers/environments/auto_stop_cron_worker.rb index fdc9490453c..de5e10a0976 100644 --- a/app/workers/environments/auto_stop_cron_worker.rb +++ b/app/workers/environments/auto_stop_cron_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Environments - class AutoStopCronWorker + class AutoStopCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/error_tracking_issue_link_worker.rb b/app/workers/error_tracking_issue_link_worker.rb index b306ecc154b..9febd5cfecc 100644 --- a/app/workers/error_tracking_issue_link_worker.rb +++ b/app/workers/error_tracking_issue_link_worker.rb @@ -5,7 +5,7 @@ # If a link to a different GitLab issue exists, a new link # will still be created, but will not be visible in Sentry # until the prior link is deleted. -class ErrorTrackingIssueLinkWorker +class ErrorTrackingIssueLinkWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExclusiveLeaseGuard include Gitlab::Utils::StrongMemoize @@ -26,8 +26,8 @@ class ErrorTrackingIssueLinkWorker logger.info("Linking Sentry issue #{sentry_issue_id} to GitLab issue #{issue.id}") sentry_client.create_issue_link(integration_id, sentry_issue_id, issue) - rescue Sentry::Client::Error - logger.info("Failed to link Sentry issue #{sentry_issue_id} to GitLab issue #{issue.id}") + rescue Sentry::Client::Error => e + logger.info("Failed to link Sentry issue #{sentry_issue_id} to GitLab issue #{issue.id} with error: #{e.message}") end end diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb index 07f516a3390..12372961250 100644 --- a/app/workers/expire_build_artifacts_worker.rb +++ b/app/workers/expire_build_artifacts_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ExpireBuildArtifactsWorker +class ExpireBuildArtifactsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index db5240d5c8e..48fd086f88f 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ExpireBuildInstanceArtifactsWorker +class ExpireBuildInstanceArtifactsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :continuous_integration diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index 0363429587e..ce27fed7fb1 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -5,7 +5,8 @@ class ExpireJobCacheWorker include PipelineQueue queue_namespace :pipeline_cache - latency_sensitive_worker! + urgency :high + idempotent! # rubocop: disable CodeReuse/ActiveRecord def perform(job_id) diff --git a/app/workers/expire_pipeline_cache_worker.rb b/app/workers/expire_pipeline_cache_worker.rb index 1d204e0a19e..1d2708cdb44 100644 --- a/app/workers/expire_pipeline_cache_worker.rb +++ b/app/workers/expire_pipeline_cache_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class ExpirePipelineCacheWorker +class ExpirePipelineCacheWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_cache - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/file_hook_worker.rb b/app/workers/file_hook_worker.rb index 24fc2d75d24..f8cdea54a17 100644 --- a/app/workers/file_hook_worker.rb +++ b/app/workers/file_hook_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class FileHookWorker +class FileHookWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: false diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb index ad119917774..37ca3af517f 100644 --- a/app/workers/git_garbage_collect_worker.rb +++ b/app/workers/git_garbage_collect_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class GitGarbageCollectWorker +class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: false diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 44e69e48694..8fbf88a1762 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -6,17 +6,14 @@ module Gitlab # number of jobs to complete, without blocking a thread. Once all jobs have # been completed this worker will advance the import process to the next # stage. - class AdvanceStageWorker + class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + include ::Gitlab::Import::AdvanceStage sidekiq_options dead: false feature_category :importers - INTERVAL = 30.seconds.to_i - - # The number of seconds to wait (while blocking the thread) before - # continuing to the next waiter. - BLOCKING_WAIT_TIME = 5 + private # The known importer stages and their corresponding Sidekiq workers. STAGES = { @@ -26,49 +23,9 @@ module Gitlab finish: Stage::FinishImportWorker }.freeze - # project_id - The ID of the project being imported. - # waiters - A Hash mapping Gitlab::JobWaiter keys to the number of - # remaining jobs. - # next_stage - The name of the next stage to start when all jobs have been - # completed. - def perform(project_id, waiters, next_stage) - return unless import_state = find_import_state(project_id) - - new_waiters = wait_for_jobs(waiters) - - if new_waiters.empty? - # We refresh the import JID here so workers importing individual - # resources (e.g. notes) don't have to do this all the time, reducing - # the pressure on Redis. We _only_ do this once all jobs are done so - # we don't get stuck forever if one or more jobs failed to notify the - # JobWaiter. - import_state.refresh_jid_expiration - - STAGES.fetch(next_stage.to_sym).perform_async(project_id) - else - self.class.perform_in(INTERVAL, project_id, new_waiters, next_stage) - end - end - - def wait_for_jobs(waiters) - waiters.each_with_object({}) do |(key, remaining), new_waiters| - waiter = JobWaiter.new(remaining, key) - - # We wait for a brief moment of time so we don't reschedule if we can - # complete the work fast enough. - waiter.wait(BLOCKING_WAIT_TIME) - - next unless waiter.jobs_remaining.positive? - - new_waiters[waiter.key] = waiter.jobs_remaining - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def find_import_state(project_id) - ProjectImportState.select(:jid).with_status(:started).find_by(project_id: project_id) + def next_stage_worker(next_stage) + STAGES.fetch(next_stage.to_sym) end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/app/workers/gitlab/github_import/import_diff_note_worker.rb b/app/workers/gitlab/github_import/import_diff_note_worker.rb index ef2a74c51c5..25fb0375692 100644 --- a/app/workers/gitlab/github_import/import_diff_note_worker.rb +++ b/app/workers/gitlab/github_import/import_diff_note_worker.rb @@ -2,7 +2,7 @@ module Gitlab module GithubImport - class ImportDiffNoteWorker + class ImportDiffNoteWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter def representation_class diff --git a/app/workers/gitlab/github_import/import_issue_worker.rb b/app/workers/gitlab/github_import/import_issue_worker.rb index 1b081ae5966..d9c496e3eb3 100644 --- a/app/workers/gitlab/github_import/import_issue_worker.rb +++ b/app/workers/gitlab/github_import/import_issue_worker.rb @@ -2,7 +2,7 @@ module Gitlab module GithubImport - class ImportIssueWorker + class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter def representation_class diff --git a/app/workers/gitlab/github_import/import_lfs_object_worker.rb b/app/workers/gitlab/github_import/import_lfs_object_worker.rb index 520c5cb091a..78f78fdb160 100644 --- a/app/workers/gitlab/github_import/import_lfs_object_worker.rb +++ b/app/workers/gitlab/github_import/import_lfs_object_worker.rb @@ -2,7 +2,7 @@ module Gitlab module GithubImport - class ImportLfsObjectWorker + class ImportLfsObjectWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter def representation_class diff --git a/app/workers/gitlab/github_import/import_note_worker.rb b/app/workers/gitlab/github_import/import_note_worker.rb index d2b4c36a5b9..d0f97a15afd 100644 --- a/app/workers/gitlab/github_import/import_note_worker.rb +++ b/app/workers/gitlab/github_import/import_note_worker.rb @@ -2,7 +2,7 @@ module Gitlab module GithubImport - class ImportNoteWorker + class ImportNoteWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter def representation_class diff --git a/app/workers/gitlab/github_import/import_pull_request_worker.rb b/app/workers/gitlab/github_import/import_pull_request_worker.rb index 62a6da152a3..ec806ad170b 100644 --- a/app/workers/gitlab/github_import/import_pull_request_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_worker.rb @@ -2,7 +2,7 @@ module Gitlab module GithubImport - class ImportPullRequestWorker + class ImportPullRequestWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter def representation_class diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb index 76723e4a61f..0ddd893d0d1 100644 --- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb +++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb @@ -2,7 +2,7 @@ module Gitlab module GithubImport - class RefreshImportJidWorker + class RefreshImportJidWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index ee64f62637e..73699a74a4a 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport module Stage - class FinishImportWorker + class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index ccfed2ae187..11c2a2ac9b4 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport module Stage - class ImportBaseDataWorker + class ImportBaseDataWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index 7007754ff2e..68b6e159fa4 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport module Stage - class ImportIssuesAndDiffNotesWorker + class ImportIssuesAndDiffNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb index 29257603a9d..a19df399969 100644 --- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport module Stage - class ImportLfsObjectsWorker + class ImportLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index ccf0013180d..49b9821cd45 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport module Stage - class ImportNotesWorker + class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index 37a7a7f4ba0..3299db5653b 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport module Stage - class ImportPullRequestsWorker + class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index b5e30470070..cb9ef1cd198 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport module Stage - class ImportRepositoryWorker + class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/import/advance_stage.rb b/app/workers/gitlab/import/advance_stage.rb new file mode 100644 index 00000000000..5c836413ae3 --- /dev/null +++ b/app/workers/gitlab/import/advance_stage.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module AdvanceStage + INTERVAL = 30.seconds.to_i + + # The number of seconds to wait (while blocking the thread) before + # continuing to the next waiter. + BLOCKING_WAIT_TIME = 5 + + # project_id - The ID of the project being imported. + # waiters - A Hash mapping Gitlab::JobWaiter keys to the number of + # remaining jobs. + # next_stage - The name of the next stage to start when all jobs have been + # completed. + def perform(project_id, waiters, next_stage) + return unless import_state = find_import_state(project_id) + + new_waiters = wait_for_jobs(waiters) + + if new_waiters.empty? + # We refresh the import JID here so workers importing individual + # resources (e.g. notes) don't have to do this all the time, reducing + # the pressure on Redis. We _only_ do this once all jobs are done so + # we don't get stuck forever if one or more jobs failed to notify the + # JobWaiter. + import_state.refresh_jid_expiration + + next_stage_worker(next_stage).perform_async(project_id) + else + self.class.perform_in(INTERVAL, project_id, new_waiters, next_stage) + end + end + + def wait_for_jobs(waiters) + waiters.each_with_object({}) do |(key, remaining), new_waiters| + waiter = JobWaiter.new(remaining, key) + + # We wait for a brief moment of time so we don't reschedule if we can + # complete the work fast enough. + waiter.wait(BLOCKING_WAIT_TIME) + + next unless waiter.jobs_remaining.positive? + + new_waiters[waiter.key] = waiter.jobs_remaining + end + end + + def find_import_state(project_id) + ProjectImportState.jid_by(project_id: project_id, status: :started) + end + + private + + def next_stage_worker(next_stage) + raise NotImplementedError + end + end + end +end diff --git a/app/workers/gitlab/phabricator_import/base_worker.rb b/app/workers/gitlab/phabricator_import/base_worker.rb index faae71d4627..82ef9e825f9 100644 --- a/app/workers/gitlab/phabricator_import/base_worker.rb +++ b/app/workers/gitlab/phabricator_import/base_worker.rb @@ -18,7 +18,7 @@ # - It marks the import as finished when all remaining jobs are done module Gitlab module PhabricatorImport - class BaseWorker + class BaseWorker # rubocop:disable Scalability/IdempotentWorker include WorkerAttributes include Gitlab::ExclusiveLeaseHelpers diff --git a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb index b5d9e80797b..1b1d7b35dd5 100644 --- a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb +++ b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab module PhabricatorImport - class ImportTasksWorker < BaseWorker + class ImportTasksWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ProjectImportOptions # This marks the project as failed after too many tries diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index bd2225e6d7c..0db794793d4 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -1,14 +1,24 @@ # frozen_string_literal: true -class GitlabShellWorker +class GitlabShellWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include Gitlab::ShellAdapter feature_category :source_code_management - latency_sensitive_worker! + urgency :high weight 2 def perform(action, *arg) + # Gitlab::Shell is being removed but we need to continue to process jobs + # enqueued in the previous release, so handle them here. + # + # See https://gitlab.com/gitlab-org/gitlab/-/issues/25095 for more details + if AuthorizedKeysWorker::PERMITTED_ACTIONS.include?(action) + AuthorizedKeysWorker.new.perform(action, *arg) + + return + end + Gitlab::GitalyClient::NamespaceService.allow do gitlab_shell.__send__(action, *arg) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb index bf0dc0fdd59..9f0cf1728dd 100644 --- a/app/workers/gitlab_usage_ping_worker.rb +++ b/app/workers/gitlab_usage_ping_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class GitlabUsagePingWorker +class GitlabUsagePingWorker # rubocop:disable Scalability/IdempotentWorker LEASE_TIMEOUT = 86400 include ApplicationWorker @@ -9,7 +9,7 @@ class GitlabUsagePingWorker include CronjobQueue # rubocop:enable Scalability/CronWorkerContext - feature_category_not_owned! + feature_category :collection # Retry for up to approximately three hours then give up. sidekiq_options retry: 10, dead: false diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb index fc751f8b612..d80a2dad7d9 100644 --- a/app/workers/group_destroy_worker.rb +++ b/app/workers/group_destroy_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class GroupDestroyWorker +class GroupDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace diff --git a/app/workers/group_export_worker.rb b/app/workers/group_export_worker.rb index a2d34e8c8bf..3e0390429d6 100644 --- a/app/workers/group_export_worker.rb +++ b/app/workers/group_export_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class GroupExportWorker +class GroupExportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace diff --git a/app/workers/group_import_worker.rb b/app/workers/group_import_worker.rb index f283eab5814..b6fc5afc28c 100644 --- a/app/workers/group_import_worker.rb +++ b/app/workers/group_import_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class GroupImportWorker +class GroupImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace diff --git a/app/workers/hashed_storage/base_worker.rb b/app/workers/hashed_storage/base_worker.rb index 1ab2108f6bb..372440996d9 100644 --- a/app/workers/hashed_storage/base_worker.rb +++ b/app/workers/hashed_storage/base_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module HashedStorage - class BaseWorker + class BaseWorker # rubocop:disable Scalability/IdempotentWorker include ExclusiveLeaseGuard include WorkerAttributes diff --git a/app/workers/hashed_storage/migrator_worker.rb b/app/workers/hashed_storage/migrator_worker.rb index 72a3faec5f4..5cbdfcb0602 100644 --- a/app/workers/hashed_storage/migrator_worker.rb +++ b/app/workers/hashed_storage/migrator_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module HashedStorage - class MigratorWorker + class MigratorWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :hashed_storage diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index 0174467923d..3ce60ce7eb6 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module HashedStorage - class ProjectMigrateWorker < BaseWorker + class ProjectMigrateWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :hashed_storage diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index 55e1d7ab23e..17b3cca83e1 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module HashedStorage - class ProjectRollbackWorker < BaseWorker + class ProjectRollbackWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :hashed_storage diff --git a/app/workers/hashed_storage/rollbacker_worker.rb b/app/workers/hashed_storage/rollbacker_worker.rb index 8babdcfb96d..a220d3b2226 100644 --- a/app/workers/hashed_storage/rollbacker_worker.rb +++ b/app/workers/hashed_storage/rollbacker_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module HashedStorage - class RollbackerWorker + class RollbackerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :hashed_storage diff --git a/app/workers/import_export_project_cleanup_worker.rb b/app/workers/import_export_project_cleanup_worker.rb index ae236fa1fcd..dd345434d08 100644 --- a/app/workers/import_export_project_cleanup_worker.rb +++ b/app/workers/import_export_project_cleanup_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ImportExportProjectCleanupWorker +class ImportExportProjectCleanupWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb index 7c5584146ca..c7b5f8cd0a7 100644 --- a/app/workers/import_issues_csv_worker.rb +++ b/app/workers/import_issues_csv_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ImportIssuesCsvWorker +class ImportIssuesCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :issue_tracking diff --git a/app/workers/incident_management/process_alert_worker.rb b/app/workers/incident_management/process_alert_worker.rb index f3d5bc5c66b..8d4294cc231 100644 --- a/app/workers/incident_management/process_alert_worker.rb +++ b/app/workers/incident_management/process_alert_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module IncidentManagement - class ProcessAlertWorker + class ProcessAlertWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :incident_management diff --git a/app/workers/invalid_gpg_signature_update_worker.rb b/app/workers/invalid_gpg_signature_update_worker.rb index e1c2eefbf0f..1fd959c8763 100644 --- a/app/workers/invalid_gpg_signature_update_worker.rb +++ b/app/workers/invalid_gpg_signature_update_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class InvalidGpgSignatureUpdateWorker +class InvalidGpgSignatureUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/irker_worker.rb b/app/workers/irker_worker.rb index a133ed6ed1b..91ab0d69ad1 100644 --- a/app/workers/irker_worker.rb +++ b/app/workers/irker_worker.rb @@ -3,7 +3,7 @@ require 'json' require 'socket' -class IrkerWorker +class IrkerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :integrations diff --git a/app/workers/issue_due_scheduler_worker.rb b/app/workers/issue_due_scheduler_worker.rb index 59027907284..d735295d046 100644 --- a/app/workers/issue_due_scheduler_worker.rb +++ b/app/workers/issue_due_scheduler_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class IssueDueSchedulerWorker +class IssueDueSchedulerWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/mail_scheduler/issue_due_worker.rb b/app/workers/mail_scheduler/issue_due_worker.rb index 6df816de71f..309d3e13477 100644 --- a/app/workers/mail_scheduler/issue_due_worker.rb +++ b/app/workers/mail_scheduler/issue_due_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module MailScheduler - class IssueDueWorker + class IssueDueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include MailSchedulerQueue diff --git a/app/workers/mail_scheduler/notification_service_worker.rb b/app/workers/mail_scheduler/notification_service_worker.rb index ec659e39b24..691af2a724d 100644 --- a/app/workers/mail_scheduler/notification_service_worker.rb +++ b/app/workers/mail_scheduler/notification_service_worker.rb @@ -3,7 +3,7 @@ require 'active_job/arguments' module MailScheduler - class NotificationServiceWorker + class NotificationServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include MailSchedulerQueue diff --git a/app/workers/merge_request_mergeability_check_worker.rb b/app/workers/merge_request_mergeability_check_worker.rb index ed35284b66c..a26c1a886f6 100644 --- a/app/workers/merge_request_mergeability_check_worker.rb +++ b/app/workers/merge_request_mergeability_check_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class MergeRequestMergeabilityCheckWorker +class MergeRequestMergeabilityCheckWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index 48bc205113f..cc5fe884aec 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -class MergeWorker +class MergeWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management - latency_sensitive_worker! + urgency :high weight 5 def perform(merge_request_id, current_user_id, params) diff --git a/app/workers/migrate_external_diffs_worker.rb b/app/workers/migrate_external_diffs_worker.rb index d248e2b5500..0a95f40aa8f 100644 --- a/app/workers/migrate_external_diffs_worker.rb +++ b/app/workers/migrate_external_diffs_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class MigrateExternalDiffsWorker +class MigrateExternalDiffsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/namespaceless_project_destroy_worker.rb b/app/workers/namespaceless_project_destroy_worker.rb index 113afc268f2..1c8054d8fbd 100644 --- a/app/workers/namespaceless_project_destroy_worker.rb +++ b/app/workers/namespaceless_project_destroy_worker.rb @@ -6,7 +6,7 @@ # used to belong to. Projects in this state should be rare. # The worker will reject doing anything for projects that *do* have a # namespace. For those use ProjectDestroyWorker instead. -class NamespacelessProjectDestroyWorker +class NamespacelessProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace diff --git a/app/workers/namespaces/prune_aggregation_schedules_worker.rb b/app/workers/namespaces/prune_aggregation_schedules_worker.rb index aeb5aa37a10..b94c8b7b4ba 100644 --- a/app/workers/namespaces/prune_aggregation_schedules_worker.rb +++ b/app/workers/namespaces/prune_aggregation_schedules_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Namespaces - class PruneAggregationSchedulesWorker + class PruneAggregationSchedulesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/namespaces/root_statistics_worker.rb b/app/workers/namespaces/root_statistics_worker.rb index fd772c8cff6..70b2510488b 100644 --- a/app/workers/namespaces/root_statistics_worker.rb +++ b/app/workers/namespaces/root_statistics_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Namespaces - class RootStatisticsWorker + class RootStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :update_namespace_statistics @@ -16,13 +16,7 @@ module Namespaces namespace.aggregation_schedule.destroy rescue ::Namespaces::StatisticsRefresherService::RefresherError, ActiveRecord::RecordNotFound => ex - log_error(namespace.full_path, ex.message) if namespace - end - - private - - def log_error(namespace_path, error_message) - Gitlab::SidekiqLogger.error("Namespace statistics can't be updated for #{namespace_path}: #{error_message}") + Gitlab::ErrorTracking.track_exception(ex, namespace_id: namespace_id, namespace: namespace&.full_path) end end end diff --git a/app/workers/namespaces/schedule_aggregation_worker.rb b/app/workers/namespaces/schedule_aggregation_worker.rb index 87e135fbf21..94343a9e378 100644 --- a/app/workers/namespaces/schedule_aggregation_worker.rb +++ b/app/workers/namespaces/schedule_aggregation_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Namespaces - class ScheduleAggregationWorker + class ScheduleAggregationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :update_namespace_statistics @@ -16,8 +16,8 @@ module Namespaces return if root_ancestor.aggregation_scheduled? Namespace::AggregationSchedule.safe_find_or_create_by!(namespace_id: root_ancestor.id) - rescue ActiveRecord::RecordNotFound - log_error(namespace_id) + rescue ActiveRecord::RecordNotFound => ex + Gitlab::ErrorTracking.track_exception(ex, namespace_id: namespace_id) end private @@ -34,9 +34,5 @@ module Namespaces Namespace::AggregationSchedule.table_exists? end - - def log_error(root_ancestor_id) - Gitlab::SidekiqLogger.error("Namespace can't be scheduled for aggregation: #{root_ancestor_id} does not exist") - end end end diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb index d696165b447..e0e28767f8d 100644 --- a/app/workers/new_issue_worker.rb +++ b/app/workers/new_issue_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class NewIssueWorker +class NewIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include NewIssuable feature_category :issue_tracking - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu weight 2 diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb index e31ddae1f13..f672d37a83e 100644 --- a/app/workers/new_merge_request_worker.rb +++ b/app/workers/new_merge_request_worker.rb @@ -1,22 +1,20 @@ # frozen_string_literal: true -class NewMergeRequestWorker +class NewMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include NewIssuable feature_category :source_code_management - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu weight 2 def perform(merge_request_id, user_id) return unless objects_found?(merge_request_id, user_id) - EventCreateService.new.open_mr(issuable, user) - NotificationService.new.new_merge_request(issuable, user) - - issuable.diffs(include_stats: false).write_cache - issuable.create_cross_references!(user) + MergeRequests::AfterCreateService + .new(issuable.target_project, user) + .execute(issuable) end def issuable_class diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index b446e376007..8ead87a9230 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -class NewNoteWorker +class NewNoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :issue_tracking - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu weight 2 diff --git a/app/workers/new_release_worker.rb b/app/workers/new_release_worker.rb index edfdb2d7aff..3c19e5f3d2b 100644 --- a/app/workers/new_release_worker.rb +++ b/app/workers/new_release_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class NewReleaseWorker +class NewReleaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker queue_namespace :notifications diff --git a/app/workers/object_pool/create_worker.rb b/app/workers/object_pool/create_worker.rb index 135b99886dc..cf87ad95077 100644 --- a/app/workers/object_pool/create_worker.rb +++ b/app/workers/object_pool/create_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ObjectPool - class CreateWorker + class CreateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ObjectPoolQueue include ExclusiveLeaseGuard diff --git a/app/workers/object_pool/destroy_worker.rb b/app/workers/object_pool/destroy_worker.rb index ca00d467d9b..d42cee59d03 100644 --- a/app/workers/object_pool/destroy_worker.rb +++ b/app/workers/object_pool/destroy_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ObjectPool - class DestroyWorker + class DestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ObjectPoolQueue diff --git a/app/workers/object_pool/join_worker.rb b/app/workers/object_pool/join_worker.rb index ddd002eabb8..f1008d3be83 100644 --- a/app/workers/object_pool/join_worker.rb +++ b/app/workers/object_pool/join_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ObjectPool - class JoinWorker + class JoinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ObjectPoolQueue diff --git a/app/workers/object_pool/schedule_join_worker.rb b/app/workers/object_pool/schedule_join_worker.rb index 647a8b72435..c00bb2967f2 100644 --- a/app/workers/object_pool/schedule_join_worker.rb +++ b/app/workers/object_pool/schedule_join_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ObjectPool - class ScheduleJoinWorker + class ScheduleJoinWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ObjectPoolQueue diff --git a/app/workers/object_storage/background_move_worker.rb b/app/workers/object_storage/background_move_worker.rb index 55f8e1c3ede..7b0a7c7ec58 100644 --- a/app/workers/object_storage/background_move_worker.rb +++ b/app/workers/object_storage/background_move_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ObjectStorage - class BackgroundMoveWorker + class BackgroundMoveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ObjectStorageQueue diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb index 01e6fdb2d3e..d9d21f2cb7e 100644 --- a/app/workers/object_storage/migrate_uploads_worker.rb +++ b/app/workers/object_storage/migrate_uploads_worker.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# rubocop:disable Scalability/IdempotentWorker module ObjectStorage class MigrateUploadsWorker include ApplicationWorker @@ -137,3 +138,4 @@ module ObjectStorage end end end +# rubocop:enable Scalability/IdempotentWorker diff --git a/app/workers/pages_domain_removal_cron_worker.rb b/app/workers/pages_domain_removal_cron_worker.rb index 1c96dd6ad8c..cb24441d2f7 100644 --- a/app/workers/pages_domain_removal_cron_worker.rb +++ b/app/workers/pages_domain_removal_cron_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PagesDomainRemovalCronWorker +class PagesDomainRemovalCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue diff --git a/app/workers/pages_domain_ssl_renewal_cron_worker.rb b/app/workers/pages_domain_ssl_renewal_cron_worker.rb index c1201b935d1..fe6d516d3cf 100644 --- a/app/workers/pages_domain_ssl_renewal_cron_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_cron_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PagesDomainSslRenewalCronWorker +class PagesDomainSslRenewalCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue diff --git a/app/workers/pages_domain_ssl_renewal_worker.rb b/app/workers/pages_domain_ssl_renewal_worker.rb index 4db7d22ef7e..561fd59d471 100644 --- a/app/workers/pages_domain_ssl_renewal_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PagesDomainSslRenewalWorker +class PagesDomainSslRenewalWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :pages diff --git a/app/workers/pages_domain_verification_cron_worker.rb b/app/workers/pages_domain_verification_cron_worker.rb index b06aa65a8e5..a30f0b981d8 100644 --- a/app/workers/pages_domain_verification_cron_worker.rb +++ b/app/workers/pages_domain_verification_cron_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PagesDomainVerificationCronWorker +class PagesDomainVerificationCronWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue diff --git a/app/workers/pages_domain_verification_worker.rb b/app/workers/pages_domain_verification_worker.rb index b0888036498..1b4d9d3994c 100644 --- a/app/workers/pages_domain_verification_worker.rb +++ b/app/workers/pages_domain_verification_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PagesDomainVerificationWorker +class PagesDomainVerificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :pages diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index 484d9053849..875f17282f9 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PagesWorker +class PagesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 diff --git a/app/workers/personal_access_tokens/expiring_worker.rb b/app/workers/personal_access_tokens/expiring_worker.rb index 84f7ce9d5d7..86240f827fc 100644 --- a/app/workers/personal_access_tokens/expiring_worker.rb +++ b/app/workers/personal_access_tokens/expiring_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module PersonalAccessTokens - class ExpiringWorker + class ExpiringWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb index 04abc9c88fd..85ecdd02fb5 100644 --- a/app/workers/pipeline_hooks_worker.rb +++ b/app/workers/pipeline_hooks_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class PipelineHooksWorker +class PipelineHooksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_hooks - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/pipeline_metrics_worker.rb b/app/workers/pipeline_metrics_worker.rb index 3830522aaa1..1eb9b4ce089 100644 --- a/app/workers/pipeline_metrics_worker.rb +++ b/app/workers/pipeline_metrics_worker.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -class PipelineMetricsWorker +class PipelineMetricsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue - latency_sensitive_worker! + urgency :high # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) diff --git a/app/workers/pipeline_notification_worker.rb b/app/workers/pipeline_notification_worker.rb index 62ecbc8a047..3336383adf7 100644 --- a/app/workers/pipeline_notification_worker.rb +++ b/app/workers/pipeline_notification_worker.rb @@ -1,19 +1,27 @@ # frozen_string_literal: true -class PipelineNotificationWorker +class PipelineNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord - def perform(pipeline_id, recipients = nil) - pipeline = Ci::Pipeline.find_by(id: pipeline_id) + def perform(pipeline_id, args = {}) + case args + when Hash + ref_status = args[:ref_status] + recipients = args[:recipients] + else # TODO: backward compatible interface, can be removed in 12.10 + recipients = args + ref_status = nil + end + pipeline = Ci::Pipeline.find_by(id: pipeline_id) return unless pipeline - NotificationService.new.pipeline_finished(pipeline, recipients) + NotificationService.new.pipeline_finished(pipeline, ref_status: ref_status, recipients: recipients) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb index 200f3619332..66a661dde71 100644 --- a/app/workers/pipeline_process_worker.rb +++ b/app/workers/pipeline_process_worker.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -class PipelineProcessWorker +class PipelineProcessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing feature_category :continuous_integration - latency_sensitive_worker! + urgency :high # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id, build_ids = nil) diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb index 8b326b9dbb6..d81b978f9b0 100644 --- a/app/workers/pipeline_schedule_worker.rb +++ b/app/workers/pipeline_schedule_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PipelineScheduleWorker +class PipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue diff --git a/app/workers/pipeline_success_worker.rb b/app/workers/pipeline_success_worker.rb index 5c24f00e0c3..d84612c52d1 100644 --- a/app/workers/pipeline_success_worker.rb +++ b/app/workers/pipeline_success_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class PipelineSuccessWorker +class PipelineSuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing - latency_sensitive_worker! + urgency :high def perform(pipeline_id) # no-op diff --git a/app/workers/pipeline_update_ci_ref_status_worker.rb b/app/workers/pipeline_update_ci_ref_status_worker.rb new file mode 100644 index 00000000000..96e14e126de --- /dev/null +++ b/app/workers/pipeline_update_ci_ref_status_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class PipelineUpdateCiRefStatusWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include PipelineQueue + + urgency :high + worker_resource_boundary :cpu + + def perform(pipeline_id) + pipeline = Ci::Pipeline.find_by_id(pipeline_id) + + return unless pipeline + + Ci::UpdateCiRefStatusService.new(pipeline).call + end +end diff --git a/app/workers/pipeline_update_worker.rb b/app/workers/pipeline_update_worker.rb index 0321ea5a6ce..7f667057af6 100644 --- a/app/workers/pipeline_update_worker.rb +++ b/app/workers/pipeline_update_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class PipelineUpdateWorker +class PipelineUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing - latency_sensitive_worker! + urgency :high def perform(pipeline_id) Ci::Pipeline.find_by_id(pipeline_id)&.update_legacy_status diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index d5038f1152b..5178fabb2d8 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -1,17 +1,17 @@ # frozen_string_literal: true -class PostReceive +class PostReceive # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu weight 5 def perform(gl_repository, identifier, changes, push_options = {}) - project, repo_type = Gitlab::GlRepository.parse(gl_repository) + container, project, repo_type = Gitlab::GlRepository.parse(gl_repository) - if project.nil? + if project.nil? && (!repo_type.snippet? || container.is_a?(ProjectSnippet)) log("Triggered hook for non-existing project with gl_repository \"#{gl_repository}\"") return false end @@ -20,12 +20,14 @@ class PostReceive # Use Sidekiq.logger so arguments can be correlated with execution # time and thread ID's. Sidekiq.logger.info "changes: #{changes.inspect}" if ENV['SIDEKIQ_LOG_ARGUMENTS'] - post_received = Gitlab::GitPostReceive.new(project, identifier, changes, push_options) + post_received = Gitlab::GitPostReceive.new(container, identifier, changes, push_options) if repo_type.wiki? - process_wiki_changes(post_received) + process_wiki_changes(post_received, container) elsif repo_type.project? - process_project_changes(post_received) + process_project_changes(post_received, container) + elsif repo_type.snippet? + process_snippet_changes(post_received, container) else # Other repos don't have hooks for now end @@ -39,24 +41,50 @@ class PostReceive end end - def process_project_changes(post_received) + def process_project_changes(post_received, project) user = identify_user(post_received) return false unless user - project = post_received.project push_options = post_received.push_options changes = post_received.changes # We only need to expire certain caches once per push - expire_caches(post_received, post_received.project.repository) - enqueue_repository_cache_update(post_received) + expire_caches(post_received, project.repository) + enqueue_project_cache_update(post_received, project) process_ref_changes(project, user, push_options: push_options, changes: changes) - update_remote_mirrors(post_received) + update_remote_mirrors(post_received, project) after_project_changes_hooks(project, user, changes.refs, changes.repository_data) end + def process_wiki_changes(post_received, project) + project.touch(:last_activity_at, :last_repository_updated_at) + project.wiki.repository.expire_statistics_caches + ProjectCacheWorker.perform_async(project.id, [], [:wiki_size]) + + user = identify_user(post_received) + return false unless user + + # We only need to expire certain caches once per push + expire_caches(post_received, project.wiki.repository) + + ::Git::WikiPushService.new(project, user, changes: post_received.changes).execute + end + + def process_snippet_changes(post_received, snippet) + user = identify_user(post_received) + + return false unless user + + # At the moment, we only expires the repository caches. + # In the future we might need to call ProjectCacheWorker + # (or the custom class we create) to update the snippet + # repository size or any other key. + # We might also need to update the repository statistics. + expire_caches(post_received, snippet.repository) + end + # Expire the repository status, branch, and tag cache once per push. def expire_caches(post_received, repository) repository.expire_status_cache if repository.empty? @@ -65,12 +93,12 @@ class PostReceive end # Schedule an update for the repository size and commit count if necessary. - def enqueue_repository_cache_update(post_received) + def enqueue_project_cache_update(post_received, project) stats_to_invalidate = [:repository_size] stats_to_invalidate << :commit_count if post_received.includes_default_branch? ProjectCacheWorker.perform_async( - post_received.project.id, + project.id, [], stats_to_invalidate, true @@ -83,10 +111,9 @@ class PostReceive Git::ProcessRefChangesService.new(project, user, params).execute end - def update_remote_mirrors(post_received) + def update_remote_mirrors(post_received, project) return unless post_received.includes_branches? || post_received.includes_tags? - project = post_received.project return unless project.has_remote_mirror? project.mark_stuck_remote_mirrors_as_failed! @@ -99,20 +126,6 @@ class PostReceive Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes) end - def process_wiki_changes(post_received) - post_received.project.touch(:last_activity_at, :last_repository_updated_at) - post_received.project.wiki.repository.expire_statistics_caches - ProjectCacheWorker.perform_async(post_received.project.id, [], [:wiki_size]) - - user = identify_user(post_received) - return false unless user - - # We only need to expire certain caches once per push - expire_caches(post_received, post_received.project.wiki.repository) - - ::Git::WikiPushService.new(post_received.project, user, changes: post_received.changes).execute - end - def log(message) Gitlab::GitLogger.error("POST-RECEIVE: #{message}") end diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index ca2896946c9..9960e812a2f 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -7,11 +7,11 @@ # result of this the workload of this worker should be kept to a bare minimum. # Consider using an extra worker if you need to add any extra (and potentially # slow) processing of commits. -class ProcessCommitWorker +class ProcessCommitWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management - latency_sensitive_worker! + urgency :high weight 3 # project_id - The ID of the project this commit belongs to. @@ -19,13 +19,12 @@ class ProcessCommitWorker # commit_hash - Hash containing commit details to use for constructing a # Commit object without having to use the Git repository. # default - The data was pushed to the default branch. - # rubocop: disable CodeReuse/ActiveRecord def perform(project_id, user_id, commit_hash, default = false) - project = Project.find_by(id: project_id) + project = Project.id_in(project_id).first return unless project - user = User.find_by(id: user_id) + user = User.id_in(user_id).first return unless user @@ -35,12 +34,11 @@ class ProcessCommitWorker process_commit_message(project, commit, user, author, default) update_issue_metrics(commit, author) end - # rubocop: enable CodeReuse/ActiveRecord def process_commit_message(project, commit, user, author, default = false) # Ignore closing references from GitLab-generated commit messages. find_closing_issues = default && !commit.merged_merge_request?(user) - closed_issues = find_closing_issues ? commit.closes_issues(user) : [] + closed_issues = find_closing_issues ? issues_to_close(project, commit, user) : [] close_issues(project, user, author, commit, closed_issues) if closed_issues.any? commit.create_cross_references!(author, closed_issues) @@ -56,6 +54,12 @@ class ProcessCommitWorker end end + def issues_to_close(project, commit, user) + Gitlab::ClosingIssueExtractor + .new(project, user) + .closed_by_message(commit.safe_message) + end + def update_issue_metrics(commit, author) mentioned_issues = commit.all_references(author).issues diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index ae1d57aa124..573f903f4e0 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Worker for updating any project specific caches. -class ProjectCacheWorker +class ProjectCacheWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - latency_sensitive_worker! + urgency :high LEASE_TIMEOUT = 15.minutes.to_i diff --git a/app/workers/project_daily_statistics_worker.rb b/app/workers/project_daily_statistics_worker.rb index 19c2fd67763..c60bee0ffdc 100644 --- a/app/workers/project_daily_statistics_worker.rb +++ b/app/workers/project_daily_statistics_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ProjectDailyStatisticsWorker +class ProjectDailyStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb index 1d20837faa2..b3e7996f4a4 100644 --- a/app/workers/project_destroy_worker.rb +++ b/app/workers/project_destroy_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ProjectDestroyWorker +class ProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 4d2cc3cd32d..aaaf70f09b5 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -1,19 +1,26 @@ # frozen_string_literal: true -class ProjectExportWorker +class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace + include ProjectExportOptions - sidekiq_options retry: 3 feature_category :importers worker_resource_boundary :memory def perform(current_user_id, project_id, after_export_strategy = {}, params = {}) current_user = User.find(current_user_id) project = Project.find(project_id) + export_job = project.export_jobs.safe_find_or_create_by(jid: self.jid) after_export = build!(after_export_strategy) + export_job&.start + ::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export) + + export_job&.finish + rescue ActiveRecord::RecordNotFound, Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError => e + logger.error("Failed to export project #{project_id}: #{e.message}") end private diff --git a/app/workers/project_service_worker.rb b/app/workers/project_service_worker.rb index 38a2a7414a5..84c3a3e52d0 100644 --- a/app/workers/project_service_worker.rb +++ b/app/workers/project_service_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ProjectServiceWorker +class ProjectServiceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options dead: false diff --git a/app/workers/project_update_repository_storage_worker.rb b/app/workers/project_update_repository_storage_worker.rb new file mode 100644 index 00000000000..ecee33e6421 --- /dev/null +++ b/app/workers/project_update_repository_storage_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ProjectUpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + feature_category :gitaly + + def perform(project_id, new_repository_storage_key) + project = Project.find(project_id) + + ::Projects::UpdateRepositoryStorageService.new(project).execute(new_repository_storage_key) + end +end diff --git a/app/workers/propagate_service_template_worker.rb b/app/workers/propagate_service_template_worker.rb index 73a2b453207..f3a6bda1821 100644 --- a/app/workers/propagate_service_template_worker.rb +++ b/app/workers/propagate_service_template_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Worker for updating any project specific caches. -class PropagateServiceTemplateWorker +class PropagateServiceTemplateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/prune_old_events_worker.rb b/app/workers/prune_old_events_worker.rb index 835c51ec846..330de4c7cba 100644 --- a/app/workers/prune_old_events_worker.rb +++ b/app/workers/prune_old_events_worker.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true -class PruneOldEventsWorker +class PruneOldEventsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context include CronjobQueue # rubocop:enable Scalability/CronWorkerContext - feature_category_not_owned! + feature_category :users DELETE_LIMIT = 10_000 diff --git a/app/workers/prune_web_hook_logs_worker.rb b/app/workers/prune_web_hook_logs_worker.rb index dd4f16a69da..a8e81a24ecd 100644 --- a/app/workers/prune_web_hook_logs_worker.rb +++ b/app/workers/prune_web_hook_logs_worker.rb @@ -2,7 +2,7 @@ # Worker that deletes a fixed number of outdated rows from the "web_hook_logs" # table. -class PruneWebHookLogsWorker +class PruneWebHookLogsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/reactive_caching_worker.rb b/app/workers/reactive_caching_worker.rb index 6f82ad83137..716b1de2bf5 100644 --- a/app/workers/reactive_caching_worker.rb +++ b/app/workers/reactive_caching_worker.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true -class ReactiveCachingWorker +class ReactiveCachingWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category_not_owned! # TODO: The reactive caching worker should be split into - # two different workers, one for latency_sensitive jobs without external dependencies - # and another worker without latency_sensitivity, but with external dependencies + # two different workers, one for high urgency jobs without external dependencies + # and another worker without high urgency, but with external dependencies # https://gitlab.com/gitlab-com/gl-infra/scalability/issues/34 # This worker should also have `worker_has_external_dependencies!` enabled - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu def perform(class_name, id, *args) diff --git a/app/workers/rebase_worker.rb b/app/workers/rebase_worker.rb index ddf5c31a1c2..2e13af5e0aa 100644 --- a/app/workers/rebase_worker.rb +++ b/app/workers/rebase_worker.rb @@ -2,7 +2,7 @@ # The RebaseWorker must be wrapped in important concurrency code, so should only # be scheduled via MergeRequest#rebase_async -class RebaseWorker +class RebaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/remote_mirror_notification_worker.rb b/app/workers/remote_mirror_notification_worker.rb index 706131d4e4b..33f5002014d 100644 --- a/app/workers/remote_mirror_notification_worker.rb +++ b/app/workers/remote_mirror_notification_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RemoteMirrorNotificationWorker +class RemoteMirrorNotificationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/remove_expired_group_links_worker.rb b/app/workers/remove_expired_group_links_worker.rb index db35dfb3ca8..8226f22837c 100644 --- a/app/workers/remove_expired_group_links_worker.rb +++ b/app/workers/remove_expired_group_links_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RemoveExpiredGroupLinksWorker +class RemoveExpiredGroupLinksWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/remove_expired_members_worker.rb b/app/workers/remove_expired_members_worker.rb index 278adee98e9..f56a6cd9fa2 100644 --- a/app/workers/remove_expired_members_worker.rb +++ b/app/workers/remove_expired_members_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RemoveExpiredMembersWorker +class RemoveExpiredMembersWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/remove_unreferenced_lfs_objects_worker.rb b/app/workers/remove_unreferenced_lfs_objects_worker.rb index 5e3998f3915..76ab23ebbd5 100644 --- a/app/workers/remove_unreferenced_lfs_objects_worker.rb +++ b/app/workers/remove_unreferenced_lfs_objects_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RemoveUnreferencedLfsObjectsWorker +class RemoveUnreferencedLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/repository_archive_cache_worker.rb b/app/workers/repository_archive_cache_worker.rb index 76e08a80c15..84f61a60953 100644 --- a/app/workers/repository_archive_cache_worker.rb +++ b/app/workers/repository_archive_cache_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RepositoryArchiveCacheWorker +class RepositoryArchiveCacheWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index 4091c30f498..3e5e6a25228 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module RepositoryCheck - class BatchWorker + class BatchWorker # rubocop:disable Scalability/IdempotentWorker prepend_if_ee('::EE::RepositoryCheck::BatchWorker') # rubocop: disable Cop/InjectEnterpriseEditionModule include ApplicationWorker diff --git a/app/workers/repository_check/clear_worker.rb b/app/workers/repository_check/clear_worker.rb index 01964c69fb2..1689b9bf251 100644 --- a/app/workers/repository_check/clear_worker.rb +++ b/app/workers/repository_check/clear_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module RepositoryCheck - class ClearWorker + class ClearWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include RepositoryCheckQueue diff --git a/app/workers/repository_check/dispatch_worker.rb b/app/workers/repository_check/dispatch_worker.rb index f68be8832eb..d7a145011fa 100644 --- a/app/workers/repository_check/dispatch_worker.rb +++ b/app/workers/repository_check/dispatch_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module RepositoryCheck - class DispatchWorker + class DispatchWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb index cadb1de356c..b25a20b3eff 100644 --- a/app/workers/repository_check/single_repository_worker.rb +++ b/app/workers/repository_check/single_repository_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module RepositoryCheck - class SingleRepositoryWorker + class SingleRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include RepositoryCheckQueue diff --git a/app/workers/repository_cleanup_worker.rb b/app/workers/repository_cleanup_worker.rb index dd2cbd42d1f..33b7223dd95 100644 --- a/app/workers/repository_cleanup_worker.rb +++ b/app/workers/repository_cleanup_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RepositoryCleanupWorker +class RepositoryCleanupWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index ba141f808a7..395fce0696c 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true -class RepositoryForkWorker +class RepositoryForkWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - include Gitlab::ShellAdapter include ProjectStartImport include ProjectImportOptions @@ -27,18 +26,8 @@ class RepositoryForkWorker Gitlab::Metrics.add_event(:fork_repository) - result = gitlab_shell.fork_repository(source_project, target_project) - - if result - link_lfs_objects(source_project, target_project) - else - raise_fork_failure( - source_project, - target_project, - 'Failed to create fork repository' - ) - end - + gitaly_fork!(source_project, target_project) + link_lfs_objects(source_project, target_project) target_project.after_import end @@ -49,10 +38,21 @@ class RepositoryForkWorker false end + def gitaly_fork!(source_project, target_project) + source_repo = source_project.repository.raw + target_repo = target_project.repository.raw + + ::Gitlab::GitalyClient::RepositoryService.new(target_repo).fork_repository(source_repo) + rescue GRPC::BadStatus => e + Gitlab::ErrorTracking.track_exception(e, source_project_id: source_project.id, target_project_id: target_project.id) + + raise_fork_failure(source_project, target_project, 'Failed to create fork repository') + end + def link_lfs_objects(source_project, target_project) Projects::LfsPointers::LfsLinkService .new(target_project) - .execute(source_project.lfs_objects_oids) + .execute(source_project.all_lfs_objects_oids) rescue Projects::LfsPointers::LfsLinkService::TooManyOidsError raise_fork_failure( source_project, diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 15677fb0a95..9f17ef467e3 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RepositoryImportWorker +class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace include ProjectStartImport diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb index 3e55ebc77ed..23a9ec1e202 100644 --- a/app/workers/repository_remove_remote_worker.rb +++ b/app/workers/repository_remove_remote_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RepositoryRemoveRemoteWorker +class RepositoryRemoveRemoteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExclusiveLeaseGuard diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb index d1dec4cb732..cfff2382f04 100644 --- a/app/workers/repository_update_remote_mirror_worker.rb +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RepositoryUpdateRemoteMirrorWorker +class RepositoryUpdateRemoteMirrorWorker # rubocop:disable Scalability/IdempotentWorker UpdateError = Class.new(StandardError) include ApplicationWorker diff --git a/app/workers/requests_profiles_worker.rb b/app/workers/requests_profiles_worker.rb index b711cb99082..106f04d9409 100644 --- a/app/workers/requests_profiles_worker.rb +++ b/app/workers/requests_profiles_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RequestsProfilesWorker +class RequestsProfilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb index f8f8a2fe7ae..7d76cbed77f 100644 --- a/app/workers/run_pipeline_schedule_worker.rb +++ b/app/workers/run_pipeline_schedule_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RunPipelineScheduleWorker +class RunPipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue diff --git a/app/workers/schedule_migrate_external_diffs_worker.rb b/app/workers/schedule_migrate_external_diffs_worker.rb index 0e3c62cf282..4e7b60c4ab7 100644 --- a/app/workers/schedule_migrate_external_diffs_worker.rb +++ b/app/workers/schedule_migrate_external_diffs_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ScheduleMigrateExternalDiffsWorker +class ScheduleMigrateExternalDiffsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext: # This schedules the `MigrateExternalDiffsWorker` diff --git a/app/workers/self_monitoring_project_create_worker.rb b/app/workers/self_monitoring_project_create_worker.rb index 429ac8aacc4..8177efb1683 100644 --- a/app/workers/self_monitoring_project_create_worker.rb +++ b/app/workers/self_monitoring_project_create_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class SelfMonitoringProjectCreateWorker +class SelfMonitoringProjectCreateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExclusiveLeaseGuard include SelfMonitoringProjectWorker diff --git a/app/workers/self_monitoring_project_delete_worker.rb b/app/workers/self_monitoring_project_delete_worker.rb index 07a7d3f6c45..4fa05d71de5 100644 --- a/app/workers/self_monitoring_project_delete_worker.rb +++ b/app/workers/self_monitoring_project_delete_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class SelfMonitoringProjectDeleteWorker +class SelfMonitoringProjectDeleteWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExclusiveLeaseGuard include SelfMonitoringProjectWorker diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb index a96c4c6dda2..aface8288e3 100644 --- a/app/workers/stage_update_worker.rb +++ b/app/workers/stage_update_worker.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class StageUpdateWorker +class StageUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing - latency_sensitive_worker! + urgency :high def perform(stage_id) Ci::Stage.find_by_id(stage_id)&.update_legacy_status diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 6e4ffa36854..b3b1ed66efc 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class StuckCiJobsWorker +class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue diff --git a/app/workers/stuck_export_jobs_worker.rb b/app/workers/stuck_export_jobs_worker.rb new file mode 100644 index 00000000000..6d8d60d2fc0 --- /dev/null +++ b/app/workers/stuck_export_jobs_worker.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# rubocop:disable Scalability/IdempotentWorker +class StuckExportJobsWorker + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker updates export states inline and does not schedule + # other jobs. + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :importers + worker_resource_boundary :cpu + + EXPORT_JOBS_EXPIRATION = 6.hours.to_i + + def perform + failed_jobs_count = mark_stuck_jobs_as_failed! + + Gitlab::Metrics.add_event(:stuck_export_jobs, + failed_jobs_count: failed_jobs_count) + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def mark_stuck_jobs_as_failed! + jids_and_ids = enqueued_exports.pluck(:jid, :id).to_h + + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) + return unless completed_jids.any? + + completed_ids = jids_and_ids.values_at(*completed_jids) + + # We select the export states again, because they may have transitioned from + # started to finished while we were looking up their Sidekiq status. + completed_jobs = enqueued_exports.where(id: completed_ids) + + Sidekiq.logger.info( + message: 'Marked stuck export jobs as failed', + job_ids: completed_jobs.map(&:jid) + ) + + completed_jobs.each do |job| + job.fail_op + end.count + end + # rubocop: enable CodeReuse/ActiveRecord + + def enqueued_exports + ProjectExportJob.with_status([:started, :queued]) + end +end +# rubocop:enable Scalability/IdempotentWorker diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index c9675417aa4..6a48b78b22c 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class StuckImportJobsWorker +class StuckImportJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker updates several import states inline and does not schedule diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb index 9214ae038a8..e0209b8237a 100644 --- a/app/workers/stuck_merge_jobs_worker.rb +++ b/app/workers/stuck_merge_jobs_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class StuckMergeJobsWorker +class StuckMergeJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext diff --git a/app/workers/system_hook_push_worker.rb b/app/workers/system_hook_push_worker.rb index fc6237f359a..ff1f2baf058 100644 --- a/app/workers/system_hook_push_worker.rb +++ b/app/workers/system_hook_push_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class SystemHookPushWorker +class SystemHookPushWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/todos_destroyer/confidential_issue_worker.rb b/app/workers/todos_destroyer/confidential_issue_worker.rb index 240a5f98ad5..b29d4168162 100644 --- a/app/workers/todos_destroyer/confidential_issue_worker.rb +++ b/app/workers/todos_destroyer/confidential_issue_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module TodosDestroyer - class ConfidentialIssueWorker + class ConfidentialIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/entity_leave_worker.rb b/app/workers/todos_destroyer/entity_leave_worker.rb index 7db3f6c84b4..558cc32d158 100644 --- a/app/workers/todos_destroyer/entity_leave_worker.rb +++ b/app/workers/todos_destroyer/entity_leave_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module TodosDestroyer - class EntityLeaveWorker + class EntityLeaveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/group_private_worker.rb b/app/workers/todos_destroyer/group_private_worker.rb index 21ec4abe478..a1943bee2ec 100644 --- a/app/workers/todos_destroyer/group_private_worker.rb +++ b/app/workers/todos_destroyer/group_private_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module TodosDestroyer - class GroupPrivateWorker + class GroupPrivateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/private_features_worker.rb b/app/workers/todos_destroyer/private_features_worker.rb index 1e68f0fd5ae..6e55467234a 100644 --- a/app/workers/todos_destroyer/private_features_worker.rb +++ b/app/workers/todos_destroyer/private_features_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module TodosDestroyer - class PrivateFeaturesWorker + class PrivateFeaturesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include TodosDestroyerQueue diff --git a/app/workers/todos_destroyer/project_private_worker.rb b/app/workers/todos_destroyer/project_private_worker.rb index 064e001530c..2a06edc666e 100644 --- a/app/workers/todos_destroyer/project_private_worker.rb +++ b/app/workers/todos_destroyer/project_private_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module TodosDestroyer - class ProjectPrivateWorker + class ProjectPrivateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include TodosDestroyerQueue diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb index 208d8b3b9b5..ee7724e0fa8 100644 --- a/app/workers/trending_projects_worker.rb +++ b/app/workers/trending_projects_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class TrendingProjectsWorker +class TrendingProjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker # rubocop:disable Scalability/CronWorkerContext # This worker does not perform work scoped to a context diff --git a/app/workers/update_external_pull_requests_worker.rb b/app/workers/update_external_pull_requests_worker.rb index e363b33f1b9..b459d26e487 100644 --- a/app/workers/update_external_pull_requests_worker.rb +++ b/app/workers/update_external_pull_requests_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class UpdateExternalPullRequestsWorker +class UpdateExternalPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/update_head_pipeline_for_merge_request_worker.rb b/app/workers/update_head_pipeline_for_merge_request_worker.rb index e069b16eb90..69698ba81bd 100644 --- a/app/workers/update_head_pipeline_for_merge_request_worker.rb +++ b/app/workers/update_head_pipeline_for_merge_request_worker.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -class UpdateHeadPipelineForMergeRequestWorker +class UpdateHeadPipelineForMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue queue_namespace :pipeline_processing feature_category :continuous_integration - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu def perform(merge_request_id) diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index ec9739e8a11..63bb6171b9c 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -class UpdateMergeRequestsWorker +class UpdateMergeRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management - latency_sensitive_worker! + urgency :high worker_resource_boundary :cpu weight 3 @@ -18,21 +18,7 @@ class UpdateMergeRequestsWorker user = User.find_by(id: user_id) return unless user - # TODO: remove this benchmarking when we have rich logging - time = Benchmark.measure do - MergeRequests::RefreshService.new(project, user).execute(oldrev, newrev, ref) - end - - args_log = [ - "elapsed=#{time.real}", - "project_id=#{project_id}", - "user_id=#{user_id}", - "oldrev=#{oldrev}", - "newrev=#{newrev}", - "ref=#{ref}" - ].join(',') - - Rails.logger.info("UpdateMergeRequestsWorker#perform #{args_log}") if time.real > LOG_TIME_THRESHOLD # rubocop:disable Gitlab/RailsLogger + MergeRequests::RefreshService.new(project, user).execute(oldrev, newrev, ref) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/update_project_statistics_worker.rb b/app/workers/update_project_statistics_worker.rb index e36cebf6f4f..336877d9f57 100644 --- a/app/workers/update_project_statistics_worker.rb +++ b/app/workers/update_project_statistics_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Worker for updating project statistics. -class UpdateProjectStatisticsWorker +class UpdateProjectStatisticsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :source_code_management diff --git a/app/workers/upload_checksum_worker.rb b/app/workers/upload_checksum_worker.rb index d35367145b8..dc2511f718c 100644 --- a/app/workers/upload_checksum_worker.rb +++ b/app/workers/upload_checksum_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class UploadChecksumWorker +class UploadChecksumWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :geo_replication diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb index 621125c8503..2e3feb1a4d1 100644 --- a/app/workers/wait_for_cluster_creation_worker.rb +++ b/app/workers/wait_for_cluster_creation_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class WaitForClusterCreationWorker +class WaitForClusterCreationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ClusterQueue diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index c3fa3162c14..6e1e7e7d62e 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class WebHookWorker +class WebHookWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :integrations diff --git a/app/workers/x509_certificate_revoke_worker.rb b/app/workers/x509_certificate_revoke_worker.rb new file mode 100644 index 00000000000..abd0e5eefa7 --- /dev/null +++ b/app/workers/x509_certificate_revoke_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class X509CertificateRevokeWorker + include ApplicationWorker + + feature_category :source_code_management + + idempotent! + + def perform(certificate_id) + return unless certificate_id + + X509Certificate.find_by_id(certificate_id).try do |certificate| + X509CertificateRevokeService.new.execute(certificate) + end + end +end |