Improve performance of expire build artifacts workerimprove-performance-of-expire-build-artifacts-worker

2 files changed, 20 insertions, 5 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index aeb35538d67..aa1f6ba13c5 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -64,8 +64,7 @@ module Ci
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
     scope :with_artifacts_archive, ->() do
-      where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
-        '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive)
+      where('EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive)
     end
 
     scope :with_existing_job_artifacts, ->(query) do
diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb
index dce812d1ae2..a5442bacfd7 100644
--- a/app/workers/expire_build_artifacts_worker.rb
+++ b/app/workers/expire_build_artifacts_worker.rb
@@ -4,14 +4,30 @@ class ExpireBuildArtifactsWorker
   include ApplicationWorker
   include CronjobQueue
 
+  REMOVAL_SIDEKIQ_JOBS_LIMIT = 1000
+
   # rubocop: disable CodeReuse/ActiveRecord
   def perform
     Rails.logger.info 'Scheduling removal of build artifacts'
 
-    build_ids = Ci::Build.with_expired_artifacts.pluck(:id)
-    build_ids = build_ids.map { |build_id| [build_id] }
+    total_removal_sidekiq_job_count = 0
+
+    Ci::Build.with_expired_artifacts
+             .include(EachBatch)
+             .each_batch(of: 100) do |relation, index|
+      build_ids = relation.pluck(:id)
+      build_ids = build_ids.map { |build_id| [build_id] }
+
+      ExpireBuildInstanceArtifactsWorker.bulk_perform_async(build_ids)
+
+      total_removal_sidekiq_job_count += build_ids.count
 
-    ExpireBuildInstanceArtifactsWorker.bulk_perform_async(build_ids)
+      ##
+      # The number of expired artifacts could be very huge with the gitlab.com scale.
+      # In order to remove artifacts gracefully even in such an environment,
+      # this cron worker schedules only the specific amount of sidekiq jobs.
+      break if total_removal_sidekiq_job_count >= REMOVAL_SIDEKIQ_JOBS_LIMIT
+    end
   end
   # rubocop: enable CodeReuse/ActiveRecord
 end