diff options
author | Tomasz Maczukin <tomasz@maczukin.pl> | 2017-02-07 23:06:16 +0100 |
---|---|---|
committer | Tomasz Maczukin <tomasz@maczukin.pl> | 2017-03-01 13:29:52 +0100 |
commit | 0ba385b363879f270c767aab6c1799a8e675a263 (patch) | |
tree | 002964ed32cb8c247682505a227e7d26a11a8283 | |
parent | a4e996d77315a9d2598e32fab42436b1a04fadcb (diff) | |
download | gitlab-ce-0ba385b363879f270c767aab6c1799a8e675a263.tar.gz |
Add exclusive lease for stuck_ci_builds_worker
-rw-r--r-- | app/workers/stuck_ci_builds_worker.rb | 13 | ||||
-rw-r--r-- | spec/workers/stuck_ci_builds_worker_spec.rb | 17 |
2 files changed, 27 insertions, 3 deletions
diff --git a/app/workers/stuck_ci_builds_worker.rb b/app/workers/stuck_ci_builds_worker.rb index b9dac807b23..4eba311d063 100644 --- a/app/workers/stuck_ci_builds_worker.rb +++ b/app/workers/stuck_ci_builds_worker.rb @@ -7,7 +7,9 @@ class StuckCiBuildsWorker BUILD_PENDING_STUCK_TIMEOUT = 1.hour def perform - Rails.logger.info 'Cleaning stuck builds' + return unless try_obtain_lease + + Rails.logger.info "#{self.class}: Cleaning stuck builds" drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT @@ -16,6 +18,13 @@ class StuckCiBuildsWorker private + def try_obtain_lease + Gitlab::ExclusiveLease.new( + 'stuck_ci_builds_worker_lease', + timeout: 30.minutes + ).try_obtain + end + def drop(status, timeout) search(status, timeout) do |build| drop_build :outdated, build, status, timeout @@ -37,7 +46,7 @@ class StuckCiBuildsWorker end def drop_build(type, build, status, timeout) - Rails.logger.info "#{self.class}: Dropping #{type.to_s} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})" + Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})" build.drop end end diff --git a/spec/workers/stuck_ci_builds_worker_spec.rb b/spec/workers/stuck_ci_builds_worker_spec.rb index e2386336447..c88a0dfa4a6 100644 --- a/spec/workers/stuck_ci_builds_worker_spec.rb +++ b/spec/workers/stuck_ci_builds_worker_spec.rb @@ -10,7 +10,10 @@ describe StuckCiBuildsWorker do build.status end - before { build.update!(status: status, updated_at: updated_at) } + before do + build.update!(status: status, updated_at: updated_at) + allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true) + end shared_examples 'build is dropped' do it 'changes status' do @@ -96,4 +99,16 @@ describe StuckCiBuildsWorker do worker.perform end end + + describe 'exclusive lease' do + let(:status) { 'running' } + let(:updated_at) { 2.days.ago } + + it 'is guard by exclusive lease' do + worker.perform + allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false) + expect(worker).not_to receive(:drop) + worker.perform + end + end end |