diff options
author | Bob Van Landuyt <bob@vanlanduyt.co> | 2017-10-19 09:30:04 +0200 |
---|---|---|
committer | Bob Van Landuyt <bob@vanlanduyt.co> | 2017-10-23 13:53:49 +0300 |
commit | 705c15d7af013f8ac9c132888dfe2e613e424dd4 (patch) | |
tree | e0bf29ca8c71447174965f0787bf9b2ffaf7149a | |
parent | 591ee4e3611dff5f6feac3a73974da4e8d5cc69f (diff) | |
download | gitlab-ce-705c15d7af013f8ac9c132888dfe2e613e424dd4.tar.gz |
Allow enabling the circuitbreaker using an env variablebvl-circuitbreaker-backoff
That way we can enable the circuitbreaker for just one host at a time.
-rw-r--r-- | doc/administration/repository_storage_paths.md | 9 | ||||
-rw-r--r-- | lib/gitlab/git/storage/circuit_breaker.rb | 11 | ||||
-rw-r--r-- | spec/lib/gitlab/git/storage/circuit_breaker_spec.rb | 15 |
3 files changed, 32 insertions, 3 deletions
diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index 2ee8d78b2f0..96f436fa7c3 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -131,6 +131,15 @@ mount is reset. **Seconds to wait for a storage access attempt:** The time in seconds GitLab will try to access storage. After this time a timeout error will be raised. +To enable the circuitbreaker for repository storage you can flip the feature flag from a rails console: + +``` +Feature.enable('git_storage_circuit_breaker') +``` + +Alternatively it can be enabled by setting `true` in the `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. +This approach would be used when enabling the circuit breaker on a single host. + When storage failures occur, this will be visible in the admin interface like this: ![failing storage](img/failing_storage.png) diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index e53171fa3c5..be7598ef011 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -54,7 +54,7 @@ module Gitlab end def perform - return yield unless Feature.enabled?('git_storage_circuit_breaker') + return yield unless enabled? check_storage_accessible! @@ -78,6 +78,15 @@ module Gitlab private + # The circuitbreaker can be enabled for the entire fleet using a Feature + # flag. + # + # Enabling it for a single host can be done setting the + # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. + def enabled? + ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') + end + def failure_info @failure_info ||= get_failure_info end diff --git a/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb index c723ab18e67..72dabca793a 100644 --- a/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb +++ b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb @@ -200,10 +200,21 @@ describe Gitlab::Git::Storage::CircuitBreaker, clean_gitlab_redis_shared_state: end context 'with the feature disabled' do - it 'returns the block without checking accessibility' do + before do stub_feature_flags(git_storage_circuit_breaker: false) + end + + it 'returns the block without checking accessibility' do + expect(circuit_breaker).not_to receive(:check_storage_accessible!) + + result = circuit_breaker.perform { 'hello' } + + expect(result).to eq('hello') + end - expect(circuit_breaker).not_to receive(:circuit_broken?) + it 'allows enabling the feature using an ENV var' do + stub_env('GIT_STORAGE_CIRCUIT_BREAKER', 'true') + expect(circuit_breaker).to receive(:check_storage_accessible!) result = circuit_breaker.perform { 'hello' } |