diff options
Diffstat (limited to 'app/workers/container_expiration_policies/cleanup_container_repository_worker.rb')
-rw-r--r-- | app/workers/container_expiration_policies/cleanup_container_repository_worker.rb | 131 |
1 files changed, 98 insertions, 33 deletions
diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb index 53220a7afed..40cc233307a 100644 --- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb +++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb @@ -3,11 +3,14 @@ module ContainerExpirationPolicies class CleanupContainerRepositoryWorker include ApplicationWorker + + sidekiq_options retry: 3 include LimitedCapacity::Worker include Gitlab::Utils::StrongMemoize queue_namespace :container_repository feature_category :container_registry + tags :exclude_from_kubernetes urgency :low worker_resource_boundary :unknown idempotent! @@ -28,7 +31,7 @@ module ContainerExpirationPolicies log_extra_metadata_on_done(:container_repository_id, container_repository.id) log_extra_metadata_on_done(:project_id, project.id) - unless allowed_to_run?(container_repository) + unless allowed_to_run? container_repository.cleanup_unscheduled! log_extra_metadata_on_done(:cleanup_status, :skipped) return @@ -39,9 +42,13 @@ module ContainerExpirationPolicies log_on_done(result) end + def max_running_jobs + return 0 unless throttling_enabled? + + ::Gitlab::CurrentSettings.container_registry_expiration_policies_worker_capacity + end + def remaining_work_count - cleanup_scheduled_count = ContainerRepository.cleanup_scheduled.count - cleanup_unfinished_count = ContainerRepository.cleanup_unfinished.count total_count = cleanup_scheduled_count + cleanup_unfinished_count log_info( @@ -53,50 +60,95 @@ module ContainerExpirationPolicies total_count end - def max_running_jobs - return 0 unless throttling_enabled? + private - ::Gitlab::CurrentSettings.current_application_settings.container_registry_expiration_policies_worker_capacity - end + def container_repository + strong_memoize(:container_repository) do + ContainerRepository.transaction do + # rubocop: disable CodeReuse/ActiveRecord + # We need a lock to prevent two workers from picking up the same row + container_repository = if loopless_enabled? + next_container_repository + else + ContainerRepository.waiting_for_cleanup + .order(:expiration_policy_cleanup_status, :expiration_policy_started_at) + .limit(1) + .lock('FOR UPDATE SKIP LOCKED') + .first + end - private + # rubocop: enable CodeReuse/ActiveRecord + container_repository&.tap(&:cleanup_ongoing!) + end + end + end - def allowed_to_run?(container_repository) - return false unless policy&.enabled && policy&.next_run_at + def next_container_repository + # rubocop: disable CodeReuse/ActiveRecord + next_one_requiring = ContainerRepository.requiring_cleanup + .order(:expiration_policy_cleanup_status, :expiration_policy_started_at) + .limit(1) + .lock('FOR UPDATE SKIP LOCKED') + .first + return next_one_requiring if next_one_requiring + + ContainerRepository.with_unfinished_cleanup + .order(:expiration_policy_started_at) + .limit(1) + .lock('FOR UPDATE SKIP LOCKED') + .first + # rubocop: enable CodeReuse/ActiveRecord + end - Time.zone.now + max_cleanup_execution_time.seconds < policy.next_run_at + def cleanup_scheduled_count + strong_memoize(:cleanup_scheduled_count) do + if loopless_enabled? + limit = max_running_jobs + 1 + ContainerExpirationPolicy.with_container_repositories + .runnable_schedules + .limit(limit) + .count + else + ContainerRepository.cleanup_scheduled.count + end + end end - def throttling_enabled? - Feature.enabled?(:container_registry_expiration_policies_throttling) + def cleanup_unfinished_count + strong_memoize(:cleanup_unfinished_count) do + if loopless_enabled? + limit = max_running_jobs + 1 + ContainerRepository.with_unfinished_cleanup + .limit(limit) + .count + else + ContainerRepository.cleanup_unfinished.count + end + end end - def max_cleanup_execution_time - ::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout + def allowed_to_run? + return false unless policy&.enabled && policy&.next_run_at + + now = Time.zone.now + + if loopless_enabled? + policy.next_run_at < now || (now + max_cleanup_execution_time.seconds < policy.next_run_at) + else + now + max_cleanup_execution_time.seconds < policy.next_run_at + end end - def policy - project.container_expiration_policy + def throttling_enabled? + Feature.enabled?(:container_registry_expiration_policies_throttling) end - def project - container_repository.project + def loopless_enabled? + Feature.enabled?(:container_registry_expiration_policies_loopless) end - def container_repository - strong_memoize(:container_repository) do - ContainerRepository.transaction do - # rubocop: disable CodeReuse/ActiveRecord - # We need a lock to prevent two workers from picking up the same row - container_repository = ContainerRepository.waiting_for_cleanup - .order(:expiration_policy_cleanup_status, :expiration_policy_started_at) - .limit(1) - .lock('FOR UPDATE SKIP LOCKED') - .first - # rubocop: enable CodeReuse/ActiveRecord - container_repository&.tap(&:cleanup_ongoing!) - end - end + def max_cleanup_execution_time + ::Gitlab::CurrentSettings.container_registry_delete_tags_service_timeout end def log_info(extra_structure) @@ -104,6 +156,11 @@ module ContainerExpirationPolicies end def log_on_done(result) + if result.error? + log_extra_metadata_on_done(:cleanup_status, :error) + log_extra_metadata_on_done(:cleanup_error_message, result.message) + end + LOG_ON_DONE_FIELDS.each do |field| value = result.payload[field] @@ -120,5 +177,13 @@ module ContainerExpirationPolicies log_extra_metadata_on_done(:cleanup_tags_service_truncated, !!truncated) log_extra_metadata_on_done(:running_jobs_count, running_jobs_count) end + + def policy + project.container_expiration_policy + end + + def project + container_repository.project + end end end |