Merge branch 'gitlab-ee-4910-exclusive-lease' into 'master'

Add rake task to cleanup any existing exclusive lease

See merge request gitlab-org/gitlab-ce!17500

4 files changed, 59 insertions, 0 deletions

diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md
index ecf92c379fd..2b4252a7b1d 100644
--- a/doc/administration/raketasks/maintenance.md
+++ b/doc/administration/raketasks/maintenance.md
@@ -240,3 +240,31 @@ sudo gitlab-rake gitlab:tcp_check[example.com,80]
 cd /home/git/gitlab
 sudo -u git -H bundle exec rake gitlab:tcp_check[example.com,80] RAILS_ENV=production
 ```
+
+## Clear exclusive lease (DANGER)
+
+GitLab uses a shared lock mechanism: `ExclusiveLease` to prevent simultaneous operations
+in a shared resource. An example is running periodic garbage collection on repositories.
+
+In very specific situations, a operation locked by an Exclusive Lease can fail without
+releasing the lock. If you can't wait for it to expire, you can run this task to manually
+clear it.
+
+To clear all exclusive leases:
+
+DANGER: **DANGER**: 
+Don't run it while GitLab or Sidekiq is running
+
+```bash
+sudo gitlab-rake gitlab:exclusive_lease:clear
+```
+
+To specify a lease `type` or lease `type + id`, specify a scope:
+
+```bash
+# to clear all leases for repository garbage collection:
+sudo gitlab-rake gitlab:exclusive_lease:clear[project_housekeeping:*]
+
+# to clear a lease for repository garbage collection in a specific project: (id=4)
+sudo gitlab-rake gitlab:exclusive_lease:clear[project_housekeeping:4]
+```
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index dbb8f317afe..12b5e240962 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -41,6 +41,16 @@ module Gitlab
       "gitlab:exclusive_lease:#{key}"
     end
 
+    # Removes any existing exclusive_lease from redis
+    # Don't run this in a live system without making sure no one is using the leases
+    def self.reset_all!(scope = '*')
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.scan_each(match: redis_shared_state_key(scope)).each do |key|
+          redis.del(key)
+        end
+      end
+    end
+
     def initialize(key, uuid: nil, timeout:)
       @redis_shared_state_key = self.class.redis_shared_state_key(key)
       @timeout = timeout
diff --git a/lib/tasks/gitlab/exclusive_lease.rake b/lib/tasks/gitlab/exclusive_lease.rake
new file mode 100644
index 00000000000..83722bf6d94
--- /dev/null
+++ b/lib/tasks/gitlab/exclusive_lease.rake
@@ -0,0 +1,9 @@
+namespace :gitlab do
+  namespace :exclusive_lease do
+    desc 'GitLab | Clear existing exclusive leases for specified scope (default: *)'
+    task :clear, [:scope] => [:environment] do |_, args|
+      args[:scope].nil? ? Gitlab::ExclusiveLease.reset_all! : Gitlab::ExclusiveLease.reset_all!(args[:scope])
+      puts 'All exclusive lease entries were removed.'
+    end
+  end
+end
diff --git a/spec/lib/gitlab/exclusive_lease_spec.rb b/spec/lib/gitlab/exclusive_lease_spec.rb
index 6193e177668..aed7d8d81ce 100644
--- a/spec/lib/gitlab/exclusive_lease_spec.rb
+++ b/spec/lib/gitlab/exclusive_lease_spec.rb
@@ -88,4 +88,16 @@ describe Gitlab::ExclusiveLease, :clean_gitlab_redis_shared_state do
       expect(lease.ttl).to be_nil
     end
   end
+
+  describe '.reset_all!' do
+    it 'removes all existing lease keys from redis' do
+      uuid = described_class.new(unique_key, timeout: 3600).try_obtain
+
+      expect(described_class.get_uuid(unique_key)).to eq(uuid)
+
+      described_class.reset_all!
+
+      expect(described_class.get_uuid(unique_key)).to be_falsey
+    end
+  end
 end