Cleanup stuck CI builds dailydrop-old-builds

3 files changed, 63 insertions, 0 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 08c85072633..e4f235fc3de 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -35,6 +35,7 @@ v 8.1.0
   - Fix duplicate repositories in GitHub import page (Stan Hu)
   - Redirect to a default path if HTTP_REFERER is not set (Stan Hu)
   - Adds ability to create directories using the web editor (Ben Ford)
+  - Cleanup stuck CI builds
 
 v 8.1.0 (unreleased)
   - Send an email to admin email when a user is reported for spam (Jonathan Rochkind)
diff --git a/app/workers/stuck_ci_builds_worker.rb b/app/workers/stuck_ci_builds_worker.rb
new file mode 100644
index 00000000000..ad02a3b16d9
--- /dev/null
+++ b/app/workers/stuck_ci_builds_worker.rb
@@ -0,0 +1,18 @@
+class StuckCiBuildsWorker
+  include Sidekiq::Worker
+  include Sidetiq::Schedulable
+
+  BUILD_STUCK_TIMEOUT = 1.day
+
+  recurrence { daily }
+
+  def perform
+    Rails.logger.info 'Cleaning stuck builds'
+
+    builds = Ci::Build.running_or_pending.where('updated_at < ?', BUILD_STUCK_TIMEOUT.ago)
+    builds.find_each(batch_size: 50).each do |build|
+      Rails.logger.debug "Dropping stuck #{build.status} build #{build.id} for runner #{build.runner_id}"
+      build.drop
+    end
+  end
+end
diff --git a/spec/workers/stuck_ci_builds_worker_spec.rb b/spec/workers/stuck_ci_builds_worker_spec.rb
new file mode 100644
index 00000000000..f9d87d97014
--- /dev/null
+++ b/spec/workers/stuck_ci_builds_worker_spec.rb
@@ -0,0 +1,44 @@
+require "spec_helper"
+
+describe StuckCiBuildsWorker do
+  let!(:build) { create :ci_build }
+
+  subject do
+    build.reload
+    build.status
+  end
+
+  %w(pending running).each do |status|
+    context "#{status} build" do
+      before do
+        build.update!(status: status)
+      end
+
+      it 'gets dropped if it was updated over 2 days ago' do
+        build.update!(updated_at: 2.day.ago)
+        StuckCiBuildsWorker.new.perform
+        is_expected.to eq('failed')
+      end
+
+      it "is still #{status}" do
+        build.update!(updated_at: 1.minute.ago)
+        StuckCiBuildsWorker.new.perform
+        is_expected.to eq(status)
+      end
+    end
+  end
+
+  %w(success failed canceled).each do |status|
+    context "#{status} build" do
+      before do
+        build.update!(status: status)
+      end
+
+      it "is still #{status}" do
+        build.update!(updated_at: 2.day.ago)
+        StuckCiBuildsWorker.new.perform
+        is_expected.to eq(status)
+      end
+    end
+  end
+end