Periodically mark projects that are stuck in importing as failed

Adds import jid to projects
Refactor middleware to set custom expiration time via sidekiq options
Add completed_jids option to sidekiq status and a few other changes

2 files changed, 53 insertions, 1 deletions

diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb
index 68a6fd76e70..b33ba2ed7c1 100644
--- a/app/workers/repository_import_worker.rb
+++ b/app/workers/repository_import_worker.rb
@@ -2,6 +2,8 @@ class RepositoryImportWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_EXPIRATION
+
   attr_accessor :project, :current_user
 
   def perform(project_id)
@@ -12,7 +14,7 @@ class RepositoryImportWorker
                               import_url: @project.import_url,
                               path: @project.path_with_namespace)
 
-    project.update_column(:import_error, nil)
+    project.update_columns(import_jid: self.jid, import_error: nil)
 
     result = Projects::ImportService.new(project, current_user).execute
 
diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb
new file mode 100644
index 00000000000..c7871d99492
--- /dev/null
+++ b/app/workers/stuck_import_jobs_worker.rb
@@ -0,0 +1,50 @@
+class StuckImportJobsWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  EXCLUSIVE_LEASE_KEY = 'fail_stuck_imports_worker_lease'.freeze
+  IMPORT_EXPIRATION = 15.hours.to_i
+
+  def perform
+    return unless try_obtain_lease
+
+    stuck_projects.find_in_batches(batch_size: 500) do |group|
+      jids = group.map(&:import_jid)
+
+      # Find the jobs that aren't currently running or that exceeded the threshold.
+      completed_jids = Gitlab::SidekiqStatus.completed_jids(jids)
+
+      if completed_jids.any?
+        completed_ids = group.select { |project| completed_jids.include?(project.import_jid) }.map(&:id)
+
+        fail_batch!(completed_jids, completed_ids)
+      end
+    end
+
+    remove_lease
+  end
+
+  private
+
+  def stuck_projects
+    Project.select('id, import_jid').with_import_status(:started).where.not(import_jid: nil)
+  end
+
+  def fail_batch!(completed_jids, completed_ids)
+    Project.where(id: completed_ids).update_all(import_status: 'failed', import_error: error_message)
+
+    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}")
+  end
+
+  def error_message
+    "Import timed out. Import took longer than #{IMPORT_EXPIRATION} seconds"
+  end
+
+  def try_obtain_lease
+    @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
+  end
+
+  def remove_lease
+    Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
+  end
+end