diff options
Diffstat (limited to 'app/workers')
29 files changed, 435 insertions, 105 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 239b66bdeb0..fb1fcb7937c 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -273,6 +273,33 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:container_registry_migration_enqueuer + :worker_name: ContainerRegistry::Migration::EnqueuerWorker + :feature_category: :container_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: cronjob:container_registry_migration_guard + :worker_name: ContainerRegistry::Migration::GuardWorker + :feature_category: :container_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: cronjob:container_registry_migration_observer + :worker_name: ContainerRegistry::Migration::ObserverWorker + :feature_category: :container_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:database_batched_background_migration :worker_name: Database::BatchedBackgroundMigrationWorker :feature_category: :database @@ -1100,8 +1127,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :needs_own_queue + :tags: [] - :name: hashed_storage:hashed_storage_project_migrate :worker_name: HashedStorage::ProjectMigrateWorker :feature_category: :source_code_management @@ -1110,8 +1136,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :needs_own_queue + :tags: [] - :name: hashed_storage:hashed_storage_project_rollback :worker_name: HashedStorage::ProjectRollbackWorker :feature_category: :source_code_management @@ -1120,8 +1145,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :needs_own_queue + :tags: [] - :name: hashed_storage:hashed_storage_rollbacker :worker_name: HashedStorage::RollbackerWorker :feature_category: :source_code_management @@ -1130,8 +1154,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :needs_own_queue + :tags: [] - :name: incident_management:incident_management_add_severity_system_note :worker_name: IncidentManagement::AddSeveritySystemNoteWorker :feature_category: :incident_management @@ -1960,6 +1983,15 @@ :weight: 1 :idempotent: :tags: [] +- :name: background_migration_ci_database + :worker_name: BackgroundMigration::CiDatabaseWorker + :feature_category: :database + :has_external_dependencies: + :urgency: :throttled + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: bulk_import :worker_name: BulkImportWorker :feature_category: :importers @@ -2303,6 +2335,15 @@ :weight: 1 :idempotent: :tags: [] +- :name: groups_update_statistics + :worker_name: Groups::UpdateStatisticsWorker + :feature_category: :source_code_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: true + :tags: [] - :name: import_issues_csv :worker_name: ImportIssuesCsvWorker :feature_category: :team_planning @@ -2564,6 +2605,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: namespaces_update_root_statistics + :worker_name: Namespaces::UpdateRootStatisticsWorker + :feature_category: :source_code_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: new_issue :worker_name: NewIssueWorker :feature_category: :team_planning @@ -2636,15 +2686,6 @@ :weight: 1 :idempotent: :tags: [] -- :name: pages_update_configuration - :worker_name: PagesUpdateConfigurationWorker - :feature_category: :pages - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 1 - :idempotent: true - :tags: [] - :name: phabricator_import_import_tasks :worker_name: Gitlab::PhabricatorImport::ImportTasksWorker :feature_category: :importers diff --git a/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb b/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb index f5327449242..8452f2a7821 100644 --- a/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb +++ b/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb @@ -20,7 +20,6 @@ module AuthorizedProjectUpdate urgency :low queue_namespace :authorized_project_update - deduplicate :until_executing, including_scheduled: true data_consistency :delayed idempotent! diff --git a/app/workers/auto_devops/disable_worker.rb b/app/workers/auto_devops/disable_worker.rb index 9ec3e5490c2..03613db3f29 100644 --- a/app/workers/auto_devops/disable_worker.rb +++ b/app/workers/auto_devops/disable_worker.rb @@ -32,8 +32,12 @@ module AutoDevops def email_receivers_for(pipeline, project) recipients = [pipeline.user&.email] - recipients << project.owner.email unless project.group - recipients.uniq.compact + + if project.personal? + recipients << project.owners.map(&:email) + end + + recipients.flatten.uniq.compact end end end diff --git a/app/workers/background_migration/ci_database_worker.rb b/app/workers/background_migration/ci_database_worker.rb new file mode 100644 index 00000000000..901d16681fd --- /dev/null +++ b/app/workers/background_migration/ci_database_worker.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module BackgroundMigration + class CiDatabaseWorker # rubocop:disable Scalability/IdempotentWorker + include SingleDatabaseWorker + + def self.tracking_database + @tracking_database ||= Gitlab::Database::CI_DATABASE_NAME + end + end +end diff --git a/app/workers/background_migration/single_database_worker.rb b/app/workers/background_migration/single_database_worker.rb index b6661d4fd14..f3a2165c41e 100644 --- a/app/workers/background_migration/single_database_worker.rb +++ b/app/workers/background_migration/single_database_worker.rb @@ -32,10 +32,6 @@ module BackgroundMigration def tracking_database raise NotImplementedError, "#{self.name} does not implement #{__method__}" end - - def unhealthy_metric_name - raise NotImplementedError, "#{self.name} does not implement #{__method__}" - end end # Performs the background migration. @@ -55,8 +51,12 @@ module BackgroundMigration private + def tracking_database + self.class.tracking_database + end + def job_coordinator - @job_coordinator ||= Gitlab::BackgroundMigration.coordinator_for_database(self.class.tracking_database) + @job_coordinator ||= Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) end def perform_with_connection(class_name, arguments, lease_attempts) @@ -91,7 +91,7 @@ module BackgroundMigration healthy_db = healthy_database? perform = lease_obtained && healthy_db - database_unhealthy_counter.increment if lease_obtained && !healthy_db + database_unhealthy_counter.increment(db_config_name: tracking_database) if lease_obtained && !healthy_db # When the DB is unhealthy or the lease can't be obtained after several tries, # then give up on the job and log a warning. Otherwise we could end up in @@ -140,7 +140,7 @@ module BackgroundMigration def database_unhealthy_counter Gitlab::Metrics.counter( - self.class.unhealthy_metric_name, + :background_migration_database_health_reschedules, 'The number of times a background migration is rescheduled because the database is unhealthy.' ) end diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index dea0d467eca..6145f34b693 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -6,8 +6,4 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker def self.tracking_database @tracking_database ||= Gitlab::BackgroundMigration::DEFAULT_TRACKING_DATABASE end - - def self.unhealthy_metric_name - @unhealthy_metric_name ||= :background_migration_database_health_reschedules - end end diff --git a/app/workers/ci/delete_objects_worker.rb b/app/workers/ci/delete_objects_worker.rb index cbcad3e8838..32c57750076 100644 --- a/app/workers/ci/delete_objects_worker.rb +++ b/app/workers/ci/delete_objects_worker.rb @@ -22,13 +22,7 @@ module Ci end def max_running_jobs - if ::Feature.enabled?(:ci_delete_objects_medium_concurrency) - 20 - elsif ::Feature.enabled?(:ci_delete_objects_high_concurrency) - 50 - else - 2 - end + 20 end private diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index 83261d9e42e..c1fec4f0196 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -13,7 +13,6 @@ module ApplicationWorker include Gitlab::SidekiqVersioning::Worker LOGGING_EXTRA_KEY = 'extra' - DEFAULT_DELAY_INTERVAL = 1 SAFE_PUSH_BULK_LIMIT = 1000 included do @@ -92,18 +91,6 @@ module ApplicationWorker validate_worker_attributes! end - def perform_async(*args) - return super if Gitlab::Database::LoadBalancing.primary_only? - - # Worker execution for workers with data_consistency set to :delayed or :sticky - # will be delayed to give replication enough time to complete - if utilizes_load_balancing_capabilities? && Feature.disabled?(:skip_scheduling_workers_for_replicas, default_enabled: :yaml) - perform_in(delay_interval, *args) - else - super - end - end - def set_queue queue_name = ::Gitlab::SidekiqConfig::WorkerRouter.global.route(self) sidekiq_options queue: queue_name # rubocop:disable Cop/SidekiqOptionsQueue @@ -194,12 +181,6 @@ module ApplicationWorker end end - protected - - def delay_interval - DEFAULT_DELAY_INTERVAL.seconds - end - private def do_push_bulk(args_list) diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb index 5fcbd74ddad..16ac61976eb 100644 --- a/app/workers/container_expiration_policy_worker.rb +++ b/app/workers/container_expiration_policy_worker.rb @@ -57,7 +57,7 @@ class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWo def perform_unthrottled with_runnable_policy(preloaded: true) do |policy| with_context(project: policy.project, - user: policy.project.owner) do |project:, user:| + user: nil) do |project:, user:| ContainerExpirationPolicyService.new(project, user) .execute(policy) end diff --git a/app/workers/container_registry/migration/enqueuer_worker.rb b/app/workers/container_registry/migration/enqueuer_worker.rb new file mode 100644 index 00000000000..5feaba870e6 --- /dev/null +++ b/app/workers/container_registry/migration/enqueuer_worker.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +module ContainerRegistry + module Migration + class EnqueuerWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + include Gitlab::Utils::StrongMemoize + + data_consistency :always + feature_category :container_registry + urgency :low + deduplicate :until_executing, including_scheduled: true + idempotent! + + def perform + return unless migration.enabled? + return unless below_capacity? + return unless waiting_time_passed? + + re_enqueue_if_capacity if handle_aborted_migration || handle_next_migration + rescue StandardError => e + Gitlab::ErrorTracking.log_exception( + e, + next_repository_id: next_repository&.id, + next_aborted_repository_id: next_aborted_repository&.id + ) + + next_repository&.abort_import + end + + private + + def handle_aborted_migration + return unless next_aborted_repository&.retry_aborted_migration + + log_extra_metadata_on_done(:container_repository_id, next_aborted_repository.id) + log_extra_metadata_on_done(:import_type, 'retry') + + true + end + + def handle_next_migration + return unless next_repository + # We return true because the repository was successfully processed (migration_state is changed) + return true if tag_count_too_high? + return unless next_repository.start_pre_import + + log_extra_metadata_on_done(:container_repository_id, next_repository.id) + log_extra_metadata_on_done(:import_type, 'next') + + true + end + + def tag_count_too_high? + return false unless next_repository.tags_count > migration.max_tags_count + + next_repository.skip_import(reason: :too_many_tags) + + true + end + + def below_capacity? + current_capacity <= maximum_capacity + end + + def waiting_time_passed? + delay = migration.enqueue_waiting_time + return true if delay == 0 + return true unless last_step_completed_repository + + last_step_completed_repository.last_import_step_done_at < Time.zone.now - delay + end + + def current_capacity + strong_memoize(:current_capacity) do + ContainerRepository.with_migration_states( + %w[pre_importing pre_import_done importing] + ).count + end + end + + def maximum_capacity + migration.capacity + end + + def next_repository + strong_memoize(:next_repository) do + ContainerRepository.ready_for_import.take # rubocop:disable CodeReuse/ActiveRecord + end + end + + def next_aborted_repository + strong_memoize(:next_aborted_repository) do + ContainerRepository.with_migration_state('import_aborted').take # rubocop:disable CodeReuse/ActiveRecord + end + end + + def last_step_completed_repository + strong_memoize(:last_step_completed_repository) do + ContainerRepository.recently_done_migration_step.first + end + end + + def migration + ::ContainerRegistry::Migration + end + + def re_enqueue_if_capacity + return unless current_capacity < maximum_capacity + + self.class.perform_async + end + end + end +end diff --git a/app/workers/container_registry/migration/guard_worker.rb b/app/workers/container_registry/migration/guard_worker.rb new file mode 100644 index 00000000000..77ae111c1cb --- /dev/null +++ b/app/workers/container_registry/migration/guard_worker.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module ContainerRegistry + module Migration + class GuardWorker + include ApplicationWorker + # This is a general worker with no context. + # It is not scoped to a project, user or group. + # We don't have a context. + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + data_consistency :always + feature_category :container_registry + urgency :low + worker_resource_boundary :unknown + deduplicate :until_executed + idempotent! + + def perform + return unless Gitlab.com? + + repositories = ::ContainerRepository.with_stale_migration(step_before_timestamp) + .limit(max_capacity) + aborts_count = 0 + long_running_migration_ids = [] + + # the #to_a is safe as the amount of entries is limited. + # In addition, we're calling #each in the next line and we don't want two different SQL queries for these two lines + log_extra_metadata_on_done(:stale_migrations_count, repositories.to_a.size) + + repositories.each do |repository| + if abortable?(repository) + repository.abort_import + aborts_count += 1 + else + long_running_migration_ids << repository.id if long_running_migration?(repository) + end + end + + log_extra_metadata_on_done(:aborted_stale_migrations_count, aborts_count) + + if long_running_migration_ids.any? + log_extra_metadata_on_done(:long_running_stale_migration_container_repository_ids, long_running_migration_ids) + end + end + + private + + # This can ping the Container Registry API. + # We loop on a set of repositories to calls this function (see #perform) + # In the worst case scenario, we have a n+1 API calls situation here. + # + # This is reasonable because the maximum amount of repositories looped + # on is `25`. See ::ContainerRegistry::Migration.capacity. + # + # TODO We can remove this n+1 situation by having a Container Registry API + # endpoint that accepts multiple repository paths at once. This is issue + # https://gitlab.com/gitlab-org/container-registry/-/issues/582 + def abortable?(repository) + # early return to save one Container Registry API request + return true unless repository.importing? || repository.pre_importing? + return true unless external_migration_in_progress?(repository) + + false + end + + def long_running_migration?(repository) + migration_start_timestamp(repository).before?(long_running_migration_threshold) + end + + def external_migration_in_progress?(repository) + status = repository.external_import_status + + (status == 'pre_import_in_progress' && repository.pre_importing?) || + (status == 'import_in_progress' && repository.importing?) + end + + def migration_start_timestamp(repository) + if repository.pre_importing? + repository.migration_pre_import_started_at + else + repository.migration_import_started_at + end + end + + def step_before_timestamp + ::ContainerRegistry::Migration.max_step_duration.seconds.ago + end + + def max_capacity + # doubling the actual capacity to prevent issues in case the capacity + # is not properly applied + ::ContainerRegistry::Migration.capacity * 2 + end + + def long_running_migration_threshold + @threshold ||= 30.minutes.ago + end + end + end +end diff --git a/app/workers/container_registry/migration/observer_worker.rb b/app/workers/container_registry/migration/observer_worker.rb new file mode 100644 index 00000000000..757c4fd11a5 --- /dev/null +++ b/app/workers/container_registry/migration/observer_worker.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module ContainerRegistry + module Migration + class ObserverWorker + include ApplicationWorker + # This worker does not perform work scoped to a context + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + COUNT_BATCH_SIZE = 50000 + + data_consistency :sticky + feature_category :container_registry + urgency :low + deduplicate :until_executed, including_scheduled: true + idempotent! + + def perform + return unless ::ContainerRegistry::Migration.enabled? + + use_replica_if_available do + ContainerRepository::MIGRATION_STATES.each do |state| + relation = ContainerRepository.with_migration_state(state) + count = ::Gitlab::Database::BatchCount.batch_count( + relation, batch_size: COUNT_BATCH_SIZE + ) + name = "#{state}_count".to_sym + log_extra_metadata_on_done(name, count) + end + end + end + + private + + def use_replica_if_available(&block) + ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) + end + end + end +end diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index 49f0222e9c9..eaa8810a78e 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -10,8 +10,6 @@ class ExpireJobCacheWorker # rubocop:disable Scalability/IdempotentWorker queue_namespace :pipeline_cache urgency :high - - deduplicate :until_executing, including_scheduled: true idempotent! def perform(job_id) diff --git a/app/workers/groups/update_statistics_worker.rb b/app/workers/groups/update_statistics_worker.rb new file mode 100644 index 00000000000..40b9e883dbb --- /dev/null +++ b/app/workers/groups/update_statistics_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# Worker for updating group statistics. +module Groups + class UpdateStatisticsWorker + include ApplicationWorker + + data_consistency :always + + sidekiq_options retry: 3 + + worker_resource_boundary :cpu + + feature_category :source_code_management + idempotent! + loggable_arguments 0, 1 + + # group_id - The ID of the group for which to flush the cache. + # statistics - An Array containing columns from NamespaceStatistics to + # refresh, if empty all columns will be refreshed + def perform(group_id, statistics = []) + group = Group.find_by_id(group_id) + + return unless group + + Groups::UpdateStatisticsService.new(group, statistics: statistics).execute + end + end +end diff --git a/app/workers/hashed_storage/migrator_worker.rb b/app/workers/hashed_storage/migrator_worker.rb index 03019ae3131..5f90b8f1009 100644 --- a/app/workers/hashed_storage/migrator_worker.rb +++ b/app/workers/hashed_storage/migrator_worker.rb @@ -11,9 +11,6 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 - tags :needs_own_queue - # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch def perform(start, finish) diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index 460aac3f2f2..01e2d6307de 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -11,9 +11,6 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 - tags :needs_own_queue - attr_reader :project_id def perform(project_id, old_disk_path = nil) diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index 91ea3dd9189..2ec323248ab 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -11,9 +11,6 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 - tags :needs_own_queue - attr_reader :project_id def perform(project_id, old_disk_path = nil) diff --git a/app/workers/hashed_storage/rollbacker_worker.rb b/app/workers/hashed_storage/rollbacker_worker.rb index d6a16b4d083..c6c4990d799 100644 --- a/app/workers/hashed_storage/rollbacker_worker.rb +++ b/app/workers/hashed_storage/rollbacker_worker.rb @@ -11,9 +11,6 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 - tags :needs_own_queue - # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch def perform(start, finish) diff --git a/app/workers/loose_foreign_keys/cleanup_worker.rb b/app/workers/loose_foreign_keys/cleanup_worker.rb index c3492fed77b..ecece92ec1b 100644 --- a/app/workers/loose_foreign_keys/cleanup_worker.rb +++ b/app/workers/loose_foreign_keys/cleanup_worker.rb @@ -12,8 +12,6 @@ module LooseForeignKeys idempotent! def perform - return if Feature.disabled?(:loose_foreign_key_cleanup, default_enabled: :yaml) - in_lock(self.class.name.underscore, ttl: ModificationTracker::MAX_RUNTIME, retries: 0) do stats = {} diff --git a/app/workers/merge_requests/update_head_pipeline_worker.rb b/app/workers/merge_requests/update_head_pipeline_worker.rb index c8dc9d1f7c8..acebf5fc767 100644 --- a/app/workers/merge_requests/update_head_pipeline_worker.rb +++ b/app/workers/merge_requests/update_head_pipeline_worker.rb @@ -2,7 +2,6 @@ module MergeRequests class UpdateHeadPipelineWorker - include ApplicationWorker include Gitlab::EventStore::Subscriber feature_category :code_review diff --git a/app/workers/namespaces/process_sync_events_worker.rb b/app/workers/namespaces/process_sync_events_worker.rb index f3c4f5bebb1..269710dd804 100644 --- a/app/workers/namespaces/process_sync_events_worker.rb +++ b/app/workers/namespaces/process_sync_events_worker.rb @@ -16,7 +16,13 @@ module Namespaces deduplicate :until_executing def perform - ::Ci::ProcessSyncEventsService.new(::Namespaces::SyncEvent, ::Ci::NamespaceMirror).execute + results = ::Ci::ProcessSyncEventsService.new( + ::Namespaces::SyncEvent, ::Ci::NamespaceMirror + ).execute + + results.each do |key, value| + log_extra_metadata_on_done(key, value) + end end end end diff --git a/app/workers/namespaces/update_root_statistics_worker.rb b/app/workers/namespaces/update_root_statistics_worker.rb new file mode 100644 index 00000000000..9fdf8e2506b --- /dev/null +++ b/app/workers/namespaces/update_root_statistics_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Namespaces + class UpdateRootStatisticsWorker + include Gitlab::EventStore::Subscriber + + data_consistency :always + + idempotent! + + feature_category :source_code_management + + def handle_event(event) + ScheduleAggregationWorker.perform_async(event.data[:namespace_id]) + end + end +end diff --git a/app/workers/pages_update_configuration_worker.rb b/app/workers/pages_update_configuration_worker.rb deleted file mode 100644 index 9c58b40e098..00000000000 --- a/app/workers/pages_update_configuration_worker.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -# TODO: remove this in 14.7 https://gitlab.com/gitlab-org/gitlab/-/issues/348582 -class PagesUpdateConfigurationWorker - include ApplicationWorker - - data_consistency :always - - sidekiq_options retry: 1 - - idempotent! - feature_category :pages - - def perform(_project_id) - # Do nothing - end -end diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb index ebda30f57d8..5a53d53ccf9 100644 --- a/app/workers/pipeline_schedule_worker.rb +++ b/app/workers/pipeline_schedule_worker.rb @@ -13,6 +13,8 @@ class PipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker def perform Ci::PipelineSchedule.runnable_schedules.preloaded.find_in_batches do |schedules| schedules.each do |schedule| + next unless schedule.project + with_context(project: schedule.project, user: schedule.owner) do Ci::PipelineScheduleService.new(schedule.project, schedule.owner).execute(schedule) end diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 4dd9a9c6fcb..e3f8c4bcd9d 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -24,8 +24,15 @@ class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker ::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export) export_job&.finish - rescue ActiveRecord::RecordNotFound, Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError => e - logger.error("Failed to export project #{project_id}: #{e.message}") + rescue ActiveRecord::RecordNotFound => e + log_failure(project_id, e) + rescue Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError => e + log_failure(project_id, e) + export_job&.finish + rescue StandardError => e + log_failure(project_id, e) + export_job&.fail_op + raise end private @@ -35,4 +42,8 @@ class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker Gitlab::ImportExport::AfterExportStrategyBuilder.build!(strategy_klass, after_export_strategy) end + + def log_failure(project_id, ex) + logger.error("Failed to export project #{project_id}: #{ex.message}") + end end diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb index cf236f8b660..d16583975fc 100644 --- a/app/workers/projects/git_garbage_collect_worker.rb +++ b/app/workers/projects/git_garbage_collect_worker.rb @@ -16,7 +16,15 @@ module Projects def before_gitaly_call(task, resource) return unless gc?(task) - ::Projects::GitDeduplicationService.new(resource).execute + # Don't block garbage collection if we can't fetch into an object pool + # due to some gRPC error because we don't want to accumulate cruft. + # See https://gitlab.com/gitlab-org/gitaly/-/issues/4022. + begin + ::Projects::GitDeduplicationService.new(resource).execute + rescue Gitlab::Git::CommandTimedOut, GRPC::Internal => e + Gitlab::ErrorTracking.track_exception(e) + end + cleanup_orphan_lfs_file_references(resource) end diff --git a/app/workers/projects/process_sync_events_worker.rb b/app/workers/projects/process_sync_events_worker.rb index b7c4b4de3d0..1330ae47a68 100644 --- a/app/workers/projects/process_sync_events_worker.rb +++ b/app/workers/projects/process_sync_events_worker.rb @@ -16,7 +16,13 @@ module Projects deduplicate :until_executing def perform - ::Ci::ProcessSyncEventsService.new(::Projects::SyncEvent, ::Ci::ProjectMirror).execute + results = ::Ci::ProcessSyncEventsService.new( + ::Projects::SyncEvent, ::Ci::ProjectMirror + ).execute + + results.each do |key, value| + log_extra_metadata_on_done(key, value) + end end end end diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb index f08d8231e43..35e3e633c70 100644 --- a/app/workers/run_pipeline_schedule_worker.rb +++ b/app/workers/run_pipeline_schedule_worker.rb @@ -15,7 +15,7 @@ class RunPipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker schedule = Ci::PipelineSchedule.find_by_id(schedule_id) user = User.find_by_id(user_id) - return unless schedule && user + return unless schedule && schedule.project && user run_pipeline_schedule(schedule, user) end diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index 952ac94d5e6..fdcd22128a3 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -6,14 +6,14 @@ class WebHookWorker include ApplicationWorker feature_category :integrations - loggable_arguments 2 + loggable_arguments 2, 3 data_consistency :delayed sidekiq_options retry: 4, dead: false urgency :low worker_has_external_dependencies! - # Webhook recursion detection properties are passed through the `data` arg. + # Webhook recursion detection properties may be passed through the `data` arg. # This will be migrated to the `params` arg over the next few releases. # See https://gitlab.com/gitlab-org/gitlab/-/issues/347389. def perform(hook_id, data, hook_name, params = {}) @@ -21,12 +21,14 @@ class WebHookWorker return unless hook data = data.with_indifferent_access + params.symbolize_keys! - # Before executing the hook, reapply any recursion detection UUID that was - # initially present in the request header so the hook can pass this same header - # value in its request. - recursion_detection_uuid = data.delete(:_gitlab_recursion_detection_request_uuid) - Gitlab::WebHooks::RecursionDetection.set_request_uuid(recursion_detection_uuid) + # TODO: Remove in 14.9 https://gitlab.com/gitlab-org/gitlab/-/issues/347389 + params[:recursion_detection_request_uuid] ||= data.delete(:_gitlab_recursion_detection_request_uuid) + + # Before executing the hook, reapply any recursion detection UUID that was initially + # present in the request header so the hook can pass this same header value in its request. + Gitlab::WebHooks::RecursionDetection.set_request_uuid(params[:recursion_detection_request_uuid]) WebHookService.new(hook, data, hook_name, jid).execute end |