diff options
Diffstat (limited to 'app/services/ci/job_artifacts/destroy_all_expired_service.rb')
-rw-r--r-- | app/services/ci/job_artifacts/destroy_all_expired_service.rb | 40 |
1 files changed, 31 insertions, 9 deletions
diff --git a/app/services/ci/job_artifacts/destroy_all_expired_service.rb b/app/services/ci/job_artifacts/destroy_all_expired_service.rb index e4f65736a58..7fa56677a0c 100644 --- a/app/services/ci/job_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/job_artifacts/destroy_all_expired_service.rb @@ -14,6 +14,7 @@ module Ci def initialize @removed_artifacts_count = 0 + @start_at = Time.current end ## @@ -25,9 +26,9 @@ module Ci def execute in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do if ::Feature.enabled?(:ci_destroy_unlocked_job_artifacts) - destroy_unlocked_job_artifacts(Time.current) + destroy_unlocked_job_artifacts else - destroy_job_artifacts_with_slow_iteration(Time.current) + destroy_job_artifacts_with_slow_iteration end end @@ -36,16 +37,37 @@ module Ci private - def destroy_unlocked_job_artifacts(start_at) + def destroy_unlocked_job_artifacts loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do - artifacts = Ci::JobArtifact.expired_before(start_at).artifact_unlocked.limit(BATCH_SIZE) + artifacts = Ci::JobArtifact.expired_before(@start_at).artifact_unlocked.limit(BATCH_SIZE) service_response = destroy_batch(artifacts) @removed_artifacts_count += service_response[:destroyed_artifacts_count] + + update_locked_status_on_unknown_artifacts if service_response[:destroyed_artifacts_count] == 0 + + # Return a truthy value here to prevent exiting #loop_until + @removed_artifacts_count end end - def destroy_job_artifacts_with_slow_iteration(start_at) - Ci::JobArtifact.expired_before(start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index| + def update_locked_status_on_unknown_artifacts + build_ids = Ci::JobArtifact.expired_before(@start_at).artifact_unknown.limit(BATCH_SIZE).distinct_job_ids + + return unless build_ids.present? + + locked_pipeline_build_ids = ::Ci::Build.with_pipeline_locked_artifacts.id_in(build_ids).pluck_primary_key + unlocked_pipeline_build_ids = build_ids - locked_pipeline_build_ids + + update_unknown_artifacts(locked_pipeline_build_ids, Ci::JobArtifact.lockeds[:artifacts_locked]) + update_unknown_artifacts(unlocked_pipeline_build_ids, Ci::JobArtifact.lockeds[:unlocked]) + end + + def update_unknown_artifacts(build_ids, locked_value) + Ci::JobArtifact.for_job_ids(build_ids).update_all(locked: locked_value) if build_ids.any? + end + + def destroy_job_artifacts_with_slow_iteration + Ci::JobArtifact.expired_before(@start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index| # For performance reasons, join with ci_pipelines after the batch is queried. # See: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47496 artifacts = relation.unlocked @@ -53,7 +75,7 @@ module Ci service_response = destroy_batch(artifacts) @removed_artifacts_count += service_response[:destroyed_artifacts_count] - break if loop_timeout?(start_at) + break if loop_timeout? break if index >= LOOP_LIMIT end end @@ -62,8 +84,8 @@ module Ci Ci::JobArtifacts::DestroyBatchService.new(artifacts).execute end - def loop_timeout?(start_at) - Time.current > start_at + LOOP_TIMEOUT + def loop_timeout? + Time.current > @start_at + LOOP_TIMEOUT end end end |