Merge branch '24240-prometheus_healthz' into 'master'

Add /-/readiness /-/liveness and /-/health_metrics endpoints to track application readiness

Closes #24240

See merge request !10416

5 files changed, 301 insertions, 0 deletions

diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb
new file mode 100644
index 00000000000..b8b6e0c3a88
--- /dev/null
+++ b/spec/controllers/health_controller_spec.rb
@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+describe HealthController do
+  include StubENV
+
+  let(:token) { current_application_settings.health_check_access_token }
+  let(:json_response) { JSON.parse(response.body) }
+
+  before do
+    stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false')
+  end
+
+  describe '#readiness' do
+    context 'authorization token provided' do
+      before do
+        request.headers['TOKEN'] = token
+      end
+
+      it 'returns proper response' do
+        get :readiness
+        expect(json_response['db_check']['status']).to eq('ok')
+        expect(json_response['redis_check']['status']).to eq('ok')
+        expect(json_response['fs_shards_check']['status']).to eq('ok')
+        expect(json_response['fs_shards_check']['labels']['shard']).to eq('default')
+      end
+    end
+
+    context 'without authorization token' do
+      it 'returns proper response' do
+        get :readiness
+        expect(response.status).to eq(404)
+      end
+    end
+  end
+
+  describe '#liveness' do
+    context 'authorization token provided' do
+      before do
+        request.headers['TOKEN'] = token
+      end
+
+      it 'returns proper response' do
+        get :liveness
+        expect(json_response['db_check']['status']).to eq('ok')
+        expect(json_response['redis_check']['status']).to eq('ok')
+        expect(json_response['fs_shards_check']['status']).to eq('ok')
+      end
+    end
+
+    context 'without authorization token' do
+      it 'returns proper response' do
+        get :liveness
+        expect(response.status).to eq(404)
+      end
+    end
+  end
+
+  describe '#metrics' do
+    context 'authorization token provided' do
+      before do
+        request.headers['TOKEN'] = token
+      end
+
+      it 'returns DB ping metrics' do
+        get :metrics
+        expect(response.body).to match(/^db_ping_timeout 0$/)
+        expect(response.body).to match(/^db_ping_success 1$/)
+        expect(response.body).to match(/^db_ping_latency [0-9\.]+$/)
+      end
+
+      it 'returns Redis ping metrics' do
+        get :metrics
+        expect(response.body).to match(/^redis_ping_timeout 0$/)
+        expect(response.body).to match(/^redis_ping_success 1$/)
+        expect(response.body).to match(/^redis_ping_latency [0-9\.]+$/)
+      end
+
+      it 'returns file system check metrics' do
+        get :metrics
+        expect(response.body).to match(/^filesystem_access_latency{shard="default"} [0-9\.]+$/)
+        expect(response.body).to match(/^filesystem_accessible{shard="default"} 1$/)
+        expect(response.body).to match(/^filesystem_write_latency{shard="default"} [0-9\.]+$/)
+        expect(response.body).to match(/^filesystem_writable{shard="default"} 1$/)
+        expect(response.body).to match(/^filesystem_read_latency{shard="default"} [0-9\.]+$/)
+        expect(response.body).to match(/^filesystem_readable{shard="default"} 1$/)
+      end
+    end
+
+    context 'without authorization token' do
+      it 'returns proper response' do
+        get :metrics
+        expect(response.status).to eq(404)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/healthchecks/db_check_spec.rb b/spec/lib/gitlab/healthchecks/db_check_spec.rb
new file mode 100644
index 00000000000..33c6c24449c
--- /dev/null
+++ b/spec/lib/gitlab/healthchecks/db_check_spec.rb
@@ -0,0 +1,6 @@
+require 'spec_helper'
+require_relative './simple_check_shared'
+
+describe Gitlab::HealthChecks::DbCheck do
+  include_examples 'simple_check', 'db_ping', 'Db', '1'
+end
diff --git a/spec/lib/gitlab/healthchecks/fs_shards_check_spec.rb b/spec/lib/gitlab/healthchecks/fs_shards_check_spec.rb
new file mode 100644
index 00000000000..4cd8cf313a5
--- /dev/null
+++ b/spec/lib/gitlab/healthchecks/fs_shards_check_spec.rb
@@ -0,0 +1,127 @@
+require 'spec_helper'
+
+describe Gitlab::HealthChecks::FsShardsCheck do
+  let(:metric_class) { Gitlab::HealthChecks::Metric }
+  let(:result_class) { Gitlab::HealthChecks::Result }
+  let(:repository_storages) { [:default] }
+  let(:tmp_dir) { Dir.mktmpdir }
+
+  let(:storages_paths) do
+    {
+      default: { path: tmp_dir }
+    }.with_indifferent_access
+  end
+
+  before do
+    allow(described_class).to receive(:repository_storages) { repository_storages }
+    allow(described_class).to receive(:storages_paths) { storages_paths }
+  end
+
+  after do
+    FileUtils.remove_entry_secure(tmp_dir) if Dir.exist?(tmp_dir)
+  end
+
+  shared_examples 'filesystem checks' do
+    describe '#readiness' do
+      subject { described_class.readiness }
+
+      context 'storage points to not existing folder' do
+        let(:storages_paths) do
+          {
+            default: { path: 'tmp/this/path/doesnt/exist' }
+          }.with_indifferent_access
+        end
+
+        it { is_expected.to include(result_class.new(false, 'cannot stat storage', shard: :default)) }
+      end
+
+      context 'storage points to directory that has both read and write rights' do
+        before do
+          FileUtils.chmod_R(0755, tmp_dir)
+        end
+
+        it { is_expected.to include(result_class.new(true, nil, shard: :default)) }
+
+        it 'cleans up files used for testing' do
+          expect(described_class).to receive(:storage_write_test).with(any_args).and_call_original
+
+          subject
+
+          expect(Dir.entries(tmp_dir).count).to eq(2)
+        end
+
+        context 'read test fails' do
+          before do
+            allow(described_class).to receive(:storage_read_test).with(any_args).and_return(false)
+          end
+
+          it { is_expected.to include(result_class.new(false, 'cannot read from storage', shard: :default)) }
+        end
+
+        context 'write test fails' do
+          before do
+            allow(described_class).to receive(:storage_write_test).with(any_args).and_return(false)
+          end
+
+          it { is_expected.to include(result_class.new(false, 'cannot write to storage', shard: :default)) }
+        end
+      end
+    end
+
+    describe '#metrics' do
+      subject { described_class.metrics }
+
+      context 'storage points to not existing folder' do
+        let(:storages_paths) do
+          {
+            default: { path: 'tmp/this/path/doesnt/exist' }
+          }.with_indifferent_access
+        end
+
+        it { is_expected.to include(metric_class.new(:filesystem_accessible, 0, shard: :default)) }
+        it { is_expected.to include(metric_class.new(:filesystem_readable, 0, shard: :default)) }
+        it { is_expected.to include(metric_class.new(:filesystem_writable, 0, shard: :default)) }
+
+        it { is_expected.to include(have_attributes(name: :filesystem_access_latency, value: be > 0, labels: { shard: :default })) }
+        it { is_expected.to include(have_attributes(name: :filesystem_read_latency, value: be > 0, labels: { shard: :default })) }
+        it { is_expected.to include(have_attributes(name: :filesystem_write_latency, value: be > 0, labels: { shard: :default })) }
+      end
+
+      context 'storage points to directory that has both read and write rights' do
+        before do
+          FileUtils.chmod_R(0755, tmp_dir)
+        end
+
+        it { is_expected.to include(metric_class.new(:filesystem_accessible, 1, shard: :default)) }
+        it { is_expected.to include(metric_class.new(:filesystem_readable, 1, shard: :default)) }
+        it { is_expected.to include(metric_class.new(:filesystem_writable, 1, shard: :default)) }
+
+        it { is_expected.to include(have_attributes(name: :filesystem_access_latency, value: be > 0, labels: { shard: :default })) }
+        it { is_expected.to include(have_attributes(name: :filesystem_read_latency, value: be > 0, labels: { shard: :default })) }
+        it { is_expected.to include(have_attributes(name: :filesystem_write_latency, value: be > 0, labels: { shard: :default })) }
+      end
+    end
+  end
+
+  context 'when popen always finds required binaries' do
+    before do
+      allow(Gitlab::Popen).to receive(:popen).and_wrap_original do |method, *args, &block|
+        begin
+          method.call(*args, &block)
+        rescue RuntimeError
+          raise 'expected not to happen'
+        end
+      end
+    end
+
+    it_behaves_like 'filesystem checks'
+  end
+
+  context 'when popen never finds required binaries' do
+    before do
+      allow(Gitlab::Popen).to receive(:popen).and_raise(Errno::ENOENT)
+    end
+
+    it_behaves_like 'filesystem checks'
+  end
+end
diff --git a/spec/lib/gitlab/healthchecks/redis_check_spec.rb b/spec/lib/gitlab/healthchecks/redis_check_spec.rb
new file mode 100644
index 00000000000..734cdcb893e
--- /dev/null
+++ b/spec/lib/gitlab/healthchecks/redis_check_spec.rb
@@ -0,0 +1,6 @@
+require 'spec_helper'
+require_relative './simple_check_shared'
+
+describe Gitlab::HealthChecks::RedisCheck do
+  include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG'
+end
diff --git a/spec/lib/gitlab/healthchecks/simple_check_shared.rb b/spec/lib/gitlab/healthchecks/simple_check_shared.rb
new file mode 100644
index 00000000000..1fa6d0faef9
--- /dev/null
+++ b/spec/lib/gitlab/healthchecks/simple_check_shared.rb
@@ -0,0 +1,66 @@
+shared_context 'simple_check' do |metrics_prefix, check_name, success_result|
+  describe '#metrics' do
+    subject { described_class.metrics }
+    context 'Check is passing' do
+      before do
+        allow(described_class).to receive(:check).and_return success_result
+      end
+
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_success", value: 1)) }
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_timeout", value: 0)) }
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_latency", value: be > 0)) }
+    end
+
+    context 'Check is misbehaving' do
+      before do
+        allow(described_class).to receive(:check).and_return 'error!'
+      end
+
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_success", value: 0)) }
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_timeout", value: 0)) }
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_latency", value: be > 0)) }
+    end
+
+    context 'Check is timeouting' do
+      before do
+        allow(described_class).to receive(:check).and_return Timeout::Error.new
+      end
+
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_success", value: 0)) }
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_timeout", value: 1)) }
+      it { is_expected.to include(have_attributes(name: "#{metrics_prefix}_latency", value: be > 0)) }
+    end
+  end
+
+  describe '#readiness' do
+    subject { described_class.readiness }
+    context 'Check returns ok' do
+      before do
+        allow(described_class).to receive(:check).and_return success_result
+      end
+
+      it { is_expected.to have_attributes(success: true) }
+    end
+
+    context 'Check is misbehaving' do
+      before do
+        allow(described_class).to receive(:check).and_return 'error!'
+      end
+
+      it { is_expected.to have_attributes(success: false, message: "unexpected #{check_name} check result: error!") }
+    end
+
+    context 'Check is timeouting' do
+      before do
+        allow(described_class).to receive(:check ).and_return Timeout::Error.new
+      end
+
+      it { is_expected.to have_attributes(success: false, message: "#{check_name} check timed out") }
+    end
+  end
+
+  describe '#liveness' do
+    subject { described_class.readiness }
+    it { is_expected.to eq(Gitlab::HealthChecks::Result.new(true)) }
+  end
+end