Merge branch 'health-check-route'

# Conflicts:
#	db/schema.rb

17 files changed, 292 insertions, 1 deletions

diff --git a/CHANGELOG b/CHANGELOG
index efb6dc6f610..ed4b1a12395 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -14,6 +14,7 @@ v 8.8.0 (unreleased)
   - Reduce delay in destroying a project from 1-minute to immediately
   - Make build status canceled if any of the jobs was canceled and none failed
   - Upgrade Sidekiq to 4.1.2
+  - Added /health_check endpoint for checking service status
   - Sanitize repo paths in new project error message
   - Bump mail_room to 0.7.0 to fix stuck IDLE connections
   - Remove future dates from contribution calendar graph.
diff --git a/Gemfile b/Gemfile
index 2854bf8a57b..96d9decb261 100644
--- a/Gemfile
+++ b/Gemfile
@@ -332,3 +332,6 @@ gem 'oauth2', '~> 1.0.0'
 
 # Soft deletion
 gem "paranoia", "~> 2.0"
+
+# Health check
+gem 'health_check', '~> 1.5.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index bc47533e5bb..ce1da92e517 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -402,6 +402,8 @@ GEM
       html2haml (>= 1.0.1)
       railties (>= 4.0.1)
     hashie (3.4.3)
+    health_check (1.5.1)
+      rails (>= 2.3.0)
     highline (1.7.8)
     hipchat (1.5.2)
       httparty
@@ -945,6 +947,7 @@ DEPENDENCIES
   grape (~> 0.13.0)
   grape-entity (~> 0.4.2)
   haml-rails (~> 0.9.0)
+  health_check (~> 1.5.1)
   hipchat (~> 1.5.0)
   html-pipeline (~> 1.11.0)
   httparty (~> 0.13.3)
diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb
index b9eb7ae7921..8c973f0e4a8 100644
--- a/app/controllers/admin/application_settings_controller.rb
+++ b/app/controllers/admin/application_settings_controller.rb
@@ -19,6 +19,12 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
     redirect_to admin_runners_path
   end
 
+  def reset_health_check_token
+    @application_setting.reset_health_check_access_token!
+    flash[:notice] = 'New health check access token has been generated!'
+    redirect_to :back
+  end
+
   def clear_repository_check_states
     RepositoryCheck::ClearWorker.perform_async
 
diff --git a/app/controllers/admin/health_check_controller.rb b/app/controllers/admin/health_check_controller.rb
new file mode 100644
index 00000000000..241c7be0ea1
--- /dev/null
+++ b/app/controllers/admin/health_check_controller.rb
@@ -0,0 +1,5 @@
+class Admin::HealthCheckController < Admin::ApplicationController
+  def show
+    @errors = HealthCheck::Utils.process_checks('standard')
+  end
+end
diff --git a/app/controllers/health_check_controller.rb b/app/controllers/health_check_controller.rb
new file mode 100644
index 00000000000..037da7d2bce
--- /dev/null
+++ b/app/controllers/health_check_controller.rb
@@ -0,0 +1,22 @@
+class HealthCheckController < HealthCheck::HealthCheckController
+  before_action :validate_health_check_access!
+
+  private
+
+  def validate_health_check_access!
+    render_404 unless token_valid?
+  end
+
+  def token_valid?
+    token = params[:token].presence || request.headers['TOKEN']
+    token.present? &&
+      ActiveSupport::SecurityUtils.variable_size_secure_compare(
+        token,
+        current_application_settings.health_check_access_token
+      )
+  end
+
+  def render_404
+    render file: Rails.root.join('public', '404'), layout: false, status: '404'
+  end
+end
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index c143cf215e6..1a10768655f 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -1,6 +1,7 @@
 class ApplicationSetting < ActiveRecord::Base
   include TokenAuthenticatable
   add_authentication_token_field :runners_registration_token
+  add_authentication_token_field :health_check_access_token
 
   CACHE_KEY = 'application_setting.last'
 
@@ -81,6 +82,7 @@ class ApplicationSetting < ActiveRecord::Base
   end
 
   before_save :ensure_runners_registration_token
+  before_save :ensure_health_check_access_token
 
   after_commit do
     Rails.cache.write(CACHE_KEY, self)
@@ -145,4 +147,8 @@ class ApplicationSetting < ActiveRecord::Base
   def runners_registration_token
     ensure_runners_registration_token!
   end
+
+  def health_check_access_token
+    ensure_health_check_access_token!
+  end
 end
diff --git a/app/views/admin/health_check/show.html.haml b/app/views/admin/health_check/show.html.haml
new file mode 100644
index 00000000000..ad79fd26d0b
--- /dev/null
+++ b/app/views/admin/health_check/show.html.haml
@@ -0,0 +1,49 @@
+- page_title "Health Check"
+
+%h3.page-title
+  Health Check
+.bs-callout.clearfix
+  .pull-left
+    %p
+  Access token is
+  %code#health-check-token= current_application_settings.health_check_access_token
+  = button_to reset_health_check_token_admin_application_settings_path,
+    method: :put, class: 'btn btn-default',
+    data: { confirm: 'Are you sure you want to reset the health check token?' } do
+    = icon('refresh')
+    Reset health check access token
+%p.light
+  Health information can be reteived as plain text, json, or xml using:
+  %ul
+    %li
+      %code= health_check_url(token: current_application_settings.health_check_access_token)
+    %li
+      %code= health_check_url(token: current_application_settings.health_check_access_token, format: :json)
+    %li
+      %code= health_check_url(token: current_application_settings.health_check_access_token, format: :xml)
+
+%p.light
+  You can also ask for the status of specific services:
+  %ul
+    %li
+      %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :cache)
+    %li
+      %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :database)
+    %li
+      %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :migrations)
+
+%hr
+.panel.panel-default
+  .panel-heading
+    Current Status:
+    - if @errors.blank?
+      = icon('circle', class: 'cgreen')
+      Healthy
+    - else
+      = icon('warning', class: 'cred')
+      Unhealthy
+  .panel-body
+    - if @errors.blank?
+      No Health Problems Detected
+    - else
+      = @errors
diff --git a/app/views/layouts/nav/_admin.html.haml b/app/views/layouts/nav/_admin.html.haml
index 280a1b93729..f292730fe45 100644
--- a/app/views/layouts/nav/_admin.html.haml
+++ b/app/views/layouts/nav/_admin.html.haml
@@ -41,6 +41,11 @@
       = icon('file-text fw')
       %span
         Logs
+  = nav_link(controller: :health_check) do
+    = link_to admin_health_check_path, title: 'Health Check' do
+      = icon('medkit fw')
+      %span
+        Health Check
   = nav_link(controller: :broadcast_messages) do
     = link_to admin_broadcast_messages_path, title: 'Messages' do
       = icon('bullhorn fw')
diff --git a/config/initializers/health_check.rb b/config/initializers/health_check.rb
new file mode 100644
index 00000000000..79e2d23ab2e
--- /dev/null
+++ b/config/initializers/health_check.rb
@@ -0,0 +1,3 @@
+HealthCheck.setup do |config|
+  config.standard_checks = ['database', 'migrations', 'cache']
+end
diff --git a/config/routes.rb b/config/routes.rb
index dd946305181..ec876eba345 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -73,6 +73,9 @@ Rails.application.routes.draw do
     mount Sidekiq::Web, at: '/admin/sidekiq', as: :sidekiq
   end
 
+  # Health check
+  get 'health_check(/:checks)'  => 'health_check#index', as: :health_check
+
   # Enable Grack support
   mount Grack::AuthSpawner, at: '/', constraints: lambda { |request| /[-\/\w\.]+\.git\//.match(request.path_info) }, via: [:get, :post, :put]
 
@@ -254,6 +257,7 @@ Rails.application.routes.draw do
     end
 
     resource :logs, only: [:show]
+    resource :health_check, controller: 'health_check', only: [:show]
     resource :background_jobs, controller: 'background_jobs', only: [:show]
 
     resources :namespaces, path: '/projects', constraints: { id: /[a-zA-Z.0-9_\-]+/ }, only: [] do
@@ -285,6 +289,7 @@ Rails.application.routes.draw do
     resource :application_settings, only: [:show, :update] do
       resources :services
       put :reset_runners_token
+      put :reset_health_check_token
       put :clear_repository_check_states
     end
 
diff --git a/db/migrate/20160509201028_add_health_check_access_token_to_application_settings.rb b/db/migrate/20160509201028_add_health_check_access_token_to_application_settings.rb
new file mode 100644
index 00000000000..9d729fec189
--- /dev/null
+++ b/db/migrate/20160509201028_add_health_check_access_token_to_application_settings.rb
@@ -0,0 +1,5 @@
+class AddHealthCheckAccessTokenToApplicationSettings < ActiveRecord::Migration
+  def change
+    add_column :application_settings, :health_check_access_token, :string
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 74facd12088..9b5aa640cb0 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160508194200) do
+ActiveRecord::Schema.define(version: 20160509201028) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -81,6 +81,7 @@ ActiveRecord::Schema.define(version: 20160508194200) do
     t.text     "shared_runners_text"
     t.integer  "metrics_packet_size",               default: 1
     t.text     "disabled_oauth_sign_in_sources"
+    t.string   "health_check_access_token"
   end
 
   create_table "audit_events", force: :cascade do |t|
diff --git a/spec/controllers/health_check_controller_spec.rb b/spec/controllers/health_check_controller_spec.rb
new file mode 100644
index 00000000000..0d8a68bb51a
--- /dev/null
+++ b/spec/controllers/health_check_controller_spec.rb
@@ -0,0 +1,105 @@
+require 'spec_helper'
+
+describe HealthCheckController do
+  let(:token) { current_application_settings.health_check_access_token }
+  let(:json_response) { JSON.parse(response.body) }
+  let(:xml_response) { Hash.from_xml(response.body)['hash'] }
+
+  describe 'GET #index' do
+    context 'when services are up but NO access token' do
+      it 'returns a not found page' do
+        get :index
+        expect(response).to be_not_found
+      end
+    end
+
+    context 'when services are up and an access token is provided' do
+      it 'supports passing the token in the header' do
+        request.headers['TOKEN'] = token
+        get :index
+        expect(response).to be_success
+        expect(response.content_type).to eq 'text/plain'
+      end
+
+      it 'supports successful plaintest response' do
+        get :index, token: token
+        expect(response).to be_success
+        expect(response.content_type).to eq 'text/plain'
+      end
+
+      it 'supports successful json response' do
+        get :index, token: token, format: :json
+        expect(response).to be_success
+        expect(response.content_type).to eq 'application/json'
+        expect(json_response['healthy']).to be true
+      end
+
+      it 'supports successful xml response' do
+        get :index, token: token, format: :xml
+        expect(response).to be_success
+        expect(response.content_type).to eq 'application/xml'
+        expect(xml_response['healthy']).to be true
+      end
+
+      it 'supports successful responses for specific checks' do
+        get :index, token: token, checks: 'email', format: :json
+        expect(response).to be_success
+        expect(response.content_type).to eq 'application/json'
+        expect(json_response['healthy']).to be true
+      end
+    end
+
+    context 'when a service is down but NO access token' do
+      it 'returns a not found page' do
+        get :index
+        expect(response).to be_not_found
+      end
+    end
+
+    context 'when a service is down and an access token is provided' do
+      before do
+        allow(HealthCheck::Utils).to receive(:process_checks).with('standard').and_return('The server is on fire')
+        allow(HealthCheck::Utils).to receive(:process_checks).with('email').and_return('Email is on fire')
+      end
+
+      it 'supports passing the token in the header' do
+        request.headers['TOKEN'] = token
+        get :index
+        expect(response.status).to eq(500)
+        expect(response.content_type).to eq 'text/plain'
+        expect(response.body).to include('The server is on fire')
+      end
+
+      it 'supports failure plaintest response' do
+        get :index, token: token
+        expect(response.status).to eq(500)
+        expect(response.content_type).to eq 'text/plain'
+        expect(response.body).to include('The server is on fire')
+      end
+
+      it 'supports failure json response' do
+        get :index, token: token, format: :json
+        expect(response.status).to eq(500)
+        expect(response.content_type).to eq 'application/json'
+        expect(json_response['healthy']).to be false
+        expect(json_response['message']).to include('The server is on fire')
+      end
+
+      it 'supports failure xml response' do
+        get :index, token: token, format: :xml
+        expect(response.status).to eq(500)
+        expect(response.content_type).to eq 'application/xml'
+        expect(xml_response['healthy']).to be false
+        expect(xml_response['message']).to include('The server is on fire')
+      end
+
+      it 'supports failure responses for specific checks' do
+        get :index, token: token, checks: 'email', format: :json
+        expect(response.status).to eq(500)
+        expect(response.content_type).to eq 'application/json'
+        expect(json_response['healthy']).to be false
+        expect(json_response['message']).to include('Email is on fire')
+      end
+    end
+  end
+end
diff --git a/spec/features/admin/admin_health_check_spec.rb b/spec/features/admin/admin_health_check_spec.rb
new file mode 100644
index 00000000000..4fde04b609b
--- /dev/null
+++ b/spec/features/admin/admin_health_check_spec.rb
@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+feature "Admin Health Check", feature: true do
+  include WaitForAjax
+
+  before do
+    login_as :admin
+  end
+
+  describe '#show' do
+    before do
+      visit admin_health_check_path
+    end
+
+    it { page.has_text? 'Health Check' }
+    it { page.has_text? 'Health information can be reteived' }
+
+    it 'has a health check access token' do
+      token = current_application_settings.health_check_access_token
+      expect(page).to have_content("Access token is #{token}")
+      expect(page).to have_selector('#health-check-token', text: token)
+    end
+
+    describe 'reload access token', js: true do
+      it 'changes the access token' do
+        orig_token = current_application_settings.health_check_access_token
+        click_button 'Reset health check access token'
+        wait_for_ajax
+        expect(find('#health-check-token').text).not_to eq orig_token
+      end
+    end
+  end
+
+  context 'when services are up' do
+    before do
+      visit admin_health_check_path
+    end
+
+    it 'shows healthy status' do
+      expect(page).to have_content('Current Status: Healthy')
+    end
+  end
+
+  context 'when a service is down' do
+    before do
+      allow(HealthCheck::Utils).to receive(:process_checks).and_return('The server is on fire')
+      visit admin_health_check_path
+    end
+
+    it 'shows unhealthy status' do
+      expect(page).to have_content('Current Status: Unhealthy')
+      expect(page).to have_content('The server is on fire')
+    end
+  end
+end
diff --git a/spec/routing/admin_routing_spec.rb b/spec/routing/admin_routing_spec.rb
index cd16a8e6322..b5ed8584c8a 100644
--- a/spec/routing/admin_routing_spec.rb
+++ b/spec/routing/admin_routing_spec.rb
@@ -118,3 +118,10 @@ describe Admin::DashboardController, "routing" do
     expect(get("/admin")).to route_to('admin/dashboard#index')
   end
 end
+
+# admin_health_check GET    /admin/health_check(.:format) admin/health_check#show
+describe Admin::HealthCheckController, "routing" do
+  it "to #show" do
+    expect(get("/admin/health_check")).to route_to('admin/health_check#show')
+  end
+end
diff --git a/spec/routing/routing_spec.rb b/spec/routing/routing_spec.rb
index 8530a2f31d5..de13c0db5d1 100644
--- a/spec/routing/routing_spec.rb
+++ b/spec/routing/routing_spec.rb
@@ -276,3 +276,13 @@ describe "Groups", "routing" do
     expect(get('/1')).to route_to('namespaces#show', id: '1')
   end
 end
+
+describe HealthCheckController, 'routing' do
+  it 'to #index' do
+    expect(get('/health_check')).to route_to('health_check#index')
+  end
+
+  it 'also supports passing checks in the url' do
+    expect(get('/health_check/email')).to route_to('health_check#index', checks: 'email')
+  end
+end