Allow enabling the circuitbreaker using an env variablebvl-circuitbreaker-backoff

That way we can enable the circuitbreaker for just one host at a time.

3 files changed, 32 insertions, 3 deletions

diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md
index 2ee8d78b2f0..96f436fa7c3 100644
--- a/doc/administration/repository_storage_paths.md
+++ b/doc/administration/repository_storage_paths.md
@@ -131,6 +131,15 @@ mount is reset.
 **Seconds to wait for a storage access attempt:** The time in seconds GitLab will
 try to access storage. After this time a timeout error will be raised.
 
+To enable the circuitbreaker for repository storage you can flip the feature flag from a rails console:
+
+```
+Feature.enable('git_storage_circuit_breaker')
+```
+
+Alternatively it can be enabled by setting `true` in the `GIT_STORAGE_CIRCUIT_BREAKER` environment variable.
+This approach would be used when enabling the circuit breaker on a single host.
+
 When storage failures occur, this will be visible in the admin interface like this:
 
 ![failing storage](img/failing_storage.png)
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
index e53171fa3c5..be7598ef011 100644
--- a/lib/gitlab/git/storage/circuit_breaker.rb
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -54,7 +54,7 @@ module Gitlab
         end
 
         def perform
-          return yield unless Feature.enabled?('git_storage_circuit_breaker')
+          return yield unless enabled?
 
           check_storage_accessible!
 
@@ -78,6 +78,15 @@ module Gitlab
 
         private
 
+        # The circuitbreaker can be enabled for the entire fleet using a Feature
+        # flag.
+        #
+        # Enabling it for a single host can be done setting the
+        # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable.
+        def enabled?
+          ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker')
+        end
+
         def failure_info
           @failure_info ||= get_failure_info
         end
diff --git a/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb
index c723ab18e67..72dabca793a 100644
--- a/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb
+++ b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb
@@ -200,10 +200,21 @@ describe Gitlab::Git::Storage::CircuitBreaker, clean_gitlab_redis_shared_state:
     end
 
     context 'with the feature disabled' do
-      it 'returns the block without checking accessibility' do
+      before do
         stub_feature_flags(git_storage_circuit_breaker: false)
+      end
+
+      it 'returns the block without checking accessibility' do
+        expect(circuit_breaker).not_to receive(:check_storage_accessible!)
+
+        result = circuit_breaker.perform { 'hello' }
+
+        expect(result).to eq('hello')
+      end
 
-        expect(circuit_breaker).not_to receive(:circuit_broken?)
+      it 'allows enabling the feature using an ENV var' do
+        stub_env('GIT_STORAGE_CIRCUIT_BREAKER', 'true')
+        expect(circuit_breaker).to receive(:check_storage_accessible!)
 
         result = circuit_breaker.perform { 'hello' }