diff options
Diffstat (limited to 'lib/gitlab/health_checks/fs_shards_check.rb')
| -rw-r--r-- | lib/gitlab/health_checks/fs_shards_check.rb | 21 |
1 files changed, 19 insertions, 2 deletions
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index 9e91c135956..eef97f54962 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -10,7 +10,9 @@ module Gitlab def readiness repository_storages.map do |storage_name| begin - if !storage_stat_test(storage_name) + if !storage_circuitbreaker_test(storage_name) + HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name) + elsif !storage_stat_test(storage_name) HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) else with_temp_file(storage_name) do |tmp_file_path| @@ -36,7 +38,8 @@ module Gitlab [ storage_stat_metrics(storage_name), storage_write_metrics(storage_name), - storage_read_metrics(storage_name) + storage_read_metrics(storage_name), + storage_circuitbreaker_metrics(storage_name) ].flatten end end @@ -121,6 +124,12 @@ module Gitlab file_contents == RANDOM_STRING end + def storage_circuitbreaker_test(storage_name) + Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" } + rescue Gitlab::Git::Storage::Inaccessible + nil + end + def storage_stat_metrics(storage_name) operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do with_timing { storage_stat_test(storage_name) } @@ -143,6 +152,14 @@ module Gitlab end end end + + def storage_circuitbreaker_metrics(storage_name) + operation_metrics(:filesystem_circuitbreaker, + :filesystem_circuitbreaker_latency_seconds, + shard: storage_name) do + with_timing { storage_circuitbreaker_test(storage_name) } + end + end end end end |