Adds exclusive lease to Git garbage collect worker.35558-only-one-garbage-collection-should-be-running-per-project-at-once

1 files changed, 32 insertions, 2 deletions

diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index c95497dfaba..ec65d3ff65e 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -5,6 +5,9 @@ class GitGarbageCollectWorker
 
   sidekiq_options retry: false
 
+  # Timeout set to 24h
+  LEASE_TIMEOUT = 86400
+
   GITALY_MIGRATED_TASKS = {
     gc: :garbage_collect,
     full_repack: :repack_full,
@@ -13,8 +16,19 @@ class GitGarbageCollectWorker
 
   def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil)
     project = Project.find(project_id)
-    task = task.to_sym
+    active_uuid = get_lease_uuid(lease_key)
+
+    if active_uuid
+      return unless active_uuid == lease_uuid
+
+      renew_lease(lease_key, active_uuid)
+    else
+      lease_uuid = try_obtain_lease(lease_key)
+
+      return unless lease_uuid
+    end
 
+    task = task.to_sym
     cmd = command(task)
     repo_path = project.repository.path_to_repo
     description = "'#{cmd.join(' ')}' in #{repo_path}"
@@ -33,11 +47,27 @@ class GitGarbageCollectWorker
     # Refresh the branch cache in case garbage collection caused a ref lookup to fail
     flush_ref_caches(project) if task == :gc
   ensure
-    Gitlab::ExclusiveLease.cancel(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present?
+    cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present?
   end
 
   private
 
+  def try_obtain_lease(key)
+    ::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain
+  end
+
+  def renew_lease(key, uuid)
+    ::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew
+  end
+
+  def cancel_lease(key, uuid)
+    ::Gitlab::ExclusiveLease.cancel(key, uuid)
+  end
+
+  def get_lease_uuid(key)
+    ::Gitlab::ExclusiveLease.get_uuid(key)
+  end
+
   ## `repository` has to be a Gitlab::Git::Repository
   def gitaly_call(task, repository)
     client = Gitlab::GitalyClient::RepositoryService.new(repository)