summaryrefslogtreecommitdiff
path: root/app/services/ci/job_artifacts/destroy_all_expired_service.rb
diff options
context:
space:
mode:
Diffstat (limited to 'app/services/ci/job_artifacts/destroy_all_expired_service.rb')
-rw-r--r--app/services/ci/job_artifacts/destroy_all_expired_service.rb40
1 files changed, 31 insertions, 9 deletions
diff --git a/app/services/ci/job_artifacts/destroy_all_expired_service.rb b/app/services/ci/job_artifacts/destroy_all_expired_service.rb
index e4f65736a58..7fa56677a0c 100644
--- a/app/services/ci/job_artifacts/destroy_all_expired_service.rb
+++ b/app/services/ci/job_artifacts/destroy_all_expired_service.rb
@@ -14,6 +14,7 @@ module Ci
def initialize
@removed_artifacts_count = 0
+ @start_at = Time.current
end
##
@@ -25,9 +26,9 @@ module Ci
def execute
in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
if ::Feature.enabled?(:ci_destroy_unlocked_job_artifacts)
- destroy_unlocked_job_artifacts(Time.current)
+ destroy_unlocked_job_artifacts
else
- destroy_job_artifacts_with_slow_iteration(Time.current)
+ destroy_job_artifacts_with_slow_iteration
end
end
@@ -36,16 +37,37 @@ module Ci
private
- def destroy_unlocked_job_artifacts(start_at)
+ def destroy_unlocked_job_artifacts
loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
- artifacts = Ci::JobArtifact.expired_before(start_at).artifact_unlocked.limit(BATCH_SIZE)
+ artifacts = Ci::JobArtifact.expired_before(@start_at).artifact_unlocked.limit(BATCH_SIZE)
service_response = destroy_batch(artifacts)
@removed_artifacts_count += service_response[:destroyed_artifacts_count]
+
+ update_locked_status_on_unknown_artifacts if service_response[:destroyed_artifacts_count] == 0
+
+ # Return a truthy value here to prevent exiting #loop_until
+ @removed_artifacts_count
end
end
- def destroy_job_artifacts_with_slow_iteration(start_at)
- Ci::JobArtifact.expired_before(start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index|
+ def update_locked_status_on_unknown_artifacts
+ build_ids = Ci::JobArtifact.expired_before(@start_at).artifact_unknown.limit(BATCH_SIZE).distinct_job_ids
+
+ return unless build_ids.present?
+
+ locked_pipeline_build_ids = ::Ci::Build.with_pipeline_locked_artifacts.id_in(build_ids).pluck_primary_key
+ unlocked_pipeline_build_ids = build_ids - locked_pipeline_build_ids
+
+ update_unknown_artifacts(locked_pipeline_build_ids, Ci::JobArtifact.lockeds[:artifacts_locked])
+ update_unknown_artifacts(unlocked_pipeline_build_ids, Ci::JobArtifact.lockeds[:unlocked])
+ end
+
+ def update_unknown_artifacts(build_ids, locked_value)
+ Ci::JobArtifact.for_job_ids(build_ids).update_all(locked: locked_value) if build_ids.any?
+ end
+
+ def destroy_job_artifacts_with_slow_iteration
+ Ci::JobArtifact.expired_before(@start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index|
# For performance reasons, join with ci_pipelines after the batch is queried.
# See: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47496
artifacts = relation.unlocked
@@ -53,7 +75,7 @@ module Ci
service_response = destroy_batch(artifacts)
@removed_artifacts_count += service_response[:destroyed_artifacts_count]
- break if loop_timeout?(start_at)
+ break if loop_timeout?
break if index >= LOOP_LIMIT
end
end
@@ -62,8 +84,8 @@ module Ci
Ci::JobArtifacts::DestroyBatchService.new(artifacts).execute
end
- def loop_timeout?(start_at)
- Time.current > start_at + LOOP_TIMEOUT
+ def loop_timeout?
+ Time.current > @start_at + LOOP_TIMEOUT
end
end
end