Merge branch '27523-make-stuck-build-detection-more-performant' into 'master'

Make stuck builds detection more performant

Closes #27523

See merge request !9025

2 files changed, 129 insertions, 57 deletions

diff --git a/spec/workers/stuck_ci_builds_worker_spec.rb b/spec/workers/stuck_ci_builds_worker_spec.rb
deleted file mode 100644
index 801fa31b45d..00000000000
--- a/spec/workers/stuck_ci_builds_worker_spec.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-require "spec_helper"
-
-describe StuckCiBuildsWorker do
-  let!(:build) { create :ci_build }
-  let(:worker) { described_class.new }
-
-  subject do
-    build.reload
-    build.status
-  end
-
-  %w(pending running).each do |status|
-    context "#{status} build" do
-      before do
-        build.update!(status: status)
-      end
-
-      it 'gets dropped if it was updated over 2 days ago' do
-        build.update!(updated_at: 2.days.ago)
-        worker.perform
-        is_expected.to eq('failed')
-      end
-
-      it "is still #{status}" do
-        build.update!(updated_at: 1.minute.ago)
-        worker.perform
-        is_expected.to eq(status)
-      end
-    end
-  end
-
-  %w(success failed canceled).each do |status|
-    context "#{status} build" do
-      before do
-        build.update!(status: status)
-      end
-
-      it "is still #{status}" do
-        build.update!(updated_at: 2.days.ago)
-        worker.perform
-        is_expected.to eq(status)
-      end
-    end
-  end
-
-  context "for deleted project" do
-    before do
-      build.update!(status: :running, updated_at: 2.days.ago)
-      build.project.update(pending_delete: true)
-    end
-
-    it "does not drop build" do
-      expect_any_instance_of(Ci::Build).not_to receive(:drop)
-      worker.perform
-    end
-  end
-end
diff --git a/spec/workers/stuck_ci_jobs_worker_spec.rb b/spec/workers/stuck_ci_jobs_worker_spec.rb
new file mode 100644
index 00000000000..8434b0c8e5b
--- /dev/null
+++ b/spec/workers/stuck_ci_jobs_worker_spec.rb
@@ -0,0 +1,129 @@
+require 'spec_helper'
+
+describe StuckCiJobsWorker do
+  let!(:runner) { create :ci_runner }
+  let!(:job) { create :ci_build, runner: runner }
+  let(:worker) { described_class.new }
+  let(:exclusive_lease_uuid) { SecureRandom.uuid }
+
+  subject do
+    job.reload
+    job.status
+  end
+
+  before do
+    job.update!(status: status, updated_at: updated_at)
+    allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid)
+  end
+
+  shared_examples 'job is dropped' do
+    it 'changes status' do
+      worker.perform
+      is_expected.to eq('failed')
+    end
+  end
+
+  shared_examples 'job is unchanged' do
+    it "doesn't change status" do
+      worker.perform
+      is_expected.to eq(status)
+    end
+  end
+
+  context 'when job is pending' do
+    let(:status) { 'pending' }
+
+    context 'when job is not stuck' do
+      before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false) }
+
+      context 'when job was not updated for more than 1 day ago' do
+        let(:updated_at) { 2.days.ago }
+        it_behaves_like 'job is dropped'
+      end
+
+      context 'when job was updated in less than 1 day ago' do
+        let(:updated_at) { 6.hours.ago }
+        it_behaves_like 'job is unchanged'
+      end
+
+      context 'when job was not updated for more than 1 hour ago' do
+        let(:updated_at) { 2.hours.ago }
+        it_behaves_like 'job is unchanged'
+      end
+    end
+
+    context 'when job is stuck' do
+      before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true) }
+
+      context 'when job was not updated for more than 1 hour ago' do
+        let(:updated_at) { 2.hours.ago }
+        it_behaves_like 'job is dropped'
+      end
+
+      context 'when job was updated in less than 1 hour ago' do
+        let(:updated_at) { 30.minutes.ago }
+        it_behaves_like 'job is unchanged'
+      end
+    end
+  end
+
+  context 'when job is running' do
+    let(:status) { 'running' }
+
+    context 'when job was not updated for more than 1 hour ago' do
+      let(:updated_at) { 2.hours.ago }
+      it_behaves_like 'job is dropped'
+    end
+
+    context 'when job was updated in less than 1 hour ago' do
+      let(:updated_at) { 30.minutes.ago }
+      it_behaves_like 'job is unchanged'
+    end
+  end
+
+  %w(success skipped failed canceled).each do |status|
+    context "when job is #{status}" do
+      let(:status) { status }
+      let(:updated_at) { 2.days.ago }
+      it_behaves_like 'job is unchanged'
+    end
+  end
+
+  context 'for deleted project' do
+    let(:status) { 'running' }
+    let(:updated_at) { 2.days.ago }
+
+    before { job.project.update(pending_delete: true) }
+
+    it 'does not drop job' do
+      expect_any_instance_of(Ci::Build).not_to receive(:drop)
+      worker.perform
+    end
+  end
+
+  describe 'exclusive lease' do
+    let(:status) { 'running' }
+    let(:updated_at) { 2.days.ago }
+    let(:worker2) { described_class.new }
+
+    it 'is guard by exclusive lease when executed concurrently' do
+      expect(worker).to receive(:drop).at_least(:once)
+      expect(worker2).not_to receive(:drop)
+      worker.perform
+      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false)
+      worker2.perform
+    end
+
+    it 'can be executed in sequence' do
+      expect(worker).to receive(:drop).at_least(:once)
+      expect(worker2).to receive(:drop).at_least(:once)
+      worker.perform
+      worker2.perform
+    end
+
+    it 'cancels exclusive lease after worker perform' do
+      expect(Gitlab::ExclusiveLease).to receive(:cancel).with(described_class::EXCLUSIVE_LEASE_KEY, exclusive_lease_uuid)
+      worker.perform
+    end
+  end
+end