diff options
Diffstat (limited to 'lib/gitlab/git/storage/circuit_breaker.rb')
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker.rb | 28 |
1 files changed, 19 insertions, 9 deletions
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 0456ad9a1f3..2ce97ff41f9 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -64,12 +64,20 @@ module Gitlab def circuit_broken? return false if no_failures? + failure_count > failure_count_threshold + end + + def backing_off? + return false if no_failures? + recent_failure = last_failure > failure_wait_time.seconds.ago - too_many_failures = failure_count > failure_count_threshold + too_many_failures = failure_count > backoff_threshold - recent_failure || too_many_failures + recent_failure && too_many_failures end + private + def failure_info @failure_info ||= get_failure_info end @@ -94,7 +102,11 @@ module Gitlab def check_storage_accessible! if circuit_broken? - raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time) + raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) + end + + if backing_off? + raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time) end unless storage_available? @@ -131,12 +143,6 @@ module Gitlab end end - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end - - private - def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) @@ -146,6 +152,10 @@ module Gitlab FailureInfo.new(last_failure, failure_count.to_i) end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" + end end end end |