Add exclusive lease for stuck_ci_builds_worker

2 files changed, 27 insertions, 3 deletions

diff --git a/app/workers/stuck_ci_builds_worker.rb b/app/workers/stuck_ci_builds_worker.rb
index b9dac807b23..4eba311d063 100644
--- a/app/workers/stuck_ci_builds_worker.rb
+++ b/app/workers/stuck_ci_builds_worker.rb
@@ -7,7 +7,9 @@ class StuckCiBuildsWorker
   BUILD_PENDING_STUCK_TIMEOUT = 1.hour
 
   def perform
-    Rails.logger.info 'Cleaning stuck builds'
+    return unless try_obtain_lease
+
+    Rails.logger.info "#{self.class}: Cleaning stuck builds"
 
     drop       :running, BUILD_RUNNING_OUTDATED_TIMEOUT
     drop       :pending, BUILD_PENDING_OUTDATED_TIMEOUT
@@ -16,6 +18,13 @@ class StuckCiBuildsWorker
 
   private
 
+  def try_obtain_lease
+    Gitlab::ExclusiveLease.new(
+      'stuck_ci_builds_worker_lease',
+      timeout: 30.minutes
+    ).try_obtain
+  end
+
   def drop(status, timeout)
     search(status, timeout) do |build|
       drop_build :outdated, build, status, timeout
@@ -37,7 +46,7 @@ class StuckCiBuildsWorker
   end
 
   def drop_build(type, build, status, timeout)
-    Rails.logger.info "#{self.class}: Dropping #{type.to_s} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})"
+    Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})"
     build.drop
   end
 end
diff --git a/spec/workers/stuck_ci_builds_worker_spec.rb b/spec/workers/stuck_ci_builds_worker_spec.rb
index e2386336447..c88a0dfa4a6 100644
--- a/spec/workers/stuck_ci_builds_worker_spec.rb
+++ b/spec/workers/stuck_ci_builds_worker_spec.rb
@@ -10,7 +10,10 @@ describe StuckCiBuildsWorker do
     build.status
   end
 
-  before { build.update!(status: status, updated_at: updated_at) }
+  before do
+    build.update!(status: status, updated_at: updated_at)
+    allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
+  end
 
   shared_examples 'build is dropped' do
     it 'changes status' do
@@ -96,4 +99,16 @@ describe StuckCiBuildsWorker do
       worker.perform
     end
   end
+
+  describe 'exclusive lease' do
+    let(:status) { 'running' }
+    let(:updated_at) { 2.days.ago }
+
+    it 'is guard by exclusive lease' do
+      worker.perform
+      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false)
+      expect(worker).not_to receive(:drop)
+      worker.perform
+    end
+  end
 end