Allow the Sidekiq queues to throttle and the factor by which to throttle them to be configurable

8 files changed, 41 insertions, 13 deletions

diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb
index a9dcf7615c4..b81842e319b 100644
--- a/app/controllers/admin/application_settings_controller.rb
+++ b/app/controllers/admin/application_settings_controller.rb
@@ -118,6 +118,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :container_registry_token_expire_delay,
       :enabled_git_access_protocol,
       :sidekiq_throttling_enabled,
+      :sidekiq_throttling_factor,
       :housekeeping_enabled,
       :housekeeping_bitmaps_enabled,
       :housekeeping_incremental_repack_period,
@@ -126,7 +127,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       repository_storages: [],
       restricted_visibility_levels: [],
       import_sources: [],
-      disabled_oauth_sign_in_sources: []
+      disabled_oauth_sign_in_sources: [],
+      sidekiq_throttling_queues: []
     )
   end
 end
diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb
index 45a567a1eba..be5e0301a43 100644
--- a/app/helpers/application_settings_helper.rb
+++ b/app/helpers/application_settings_helper.rb
@@ -100,4 +100,8 @@ module ApplicationSettingsHelper
 
     options_for_select(options, @application_setting.repository_storages)
   end
+
+  def sidekiq_queue_options_for_select
+    options_for_select(Sidekiq::Queue.all.map(&:name), @application_setting.sidekiq_throttling_queues)
+  end
 end
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index b728083e91c..075e4f4fc9d 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -19,6 +19,7 @@ class ApplicationSetting < ActiveRecord::Base
   serialize :domain_whitelist, Array
   serialize :domain_blacklist, Array
   serialize :repository_storages
+  serialize :sidekiq_throttling_queues
 
   cache_markdown_field :sign_in_text
   cache_markdown_field :help_page_text
@@ -85,6 +86,15 @@ class ApplicationSetting < ActiveRecord::Base
             presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' },
             if: :domain_blacklist_enabled?
 
+  validates :sidekiq_throttling_factor,
+            numericality: { greater_than: 0, less_than: 1 },
+            presence: { message: 'Throttling factor cannot be empty if Sidekiq Throttling is enabled.' },
+            if: :sidekiq_throttling_enabled?
+
+  validates :sidekiq_throttling_queues,
+            presence: { message: 'Queues to throttle cannot be empty if Sidekiq Throttling is enabled.' },
+            if: :sidekiq_throttling_enabled?
+
   validates :housekeeping_incremental_repack_period,
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml
index 01a14accbba..9b1b3f0e16e 100644
--- a/app/views/admin/application_settings/_form.html.haml
+++ b/app/views/admin/application_settings/_form.html.haml
@@ -295,6 +295,18 @@
             Enable Sidekiq Job Throttling
           .help-block
             Limit the amount of resources slow running jobs are assigned.
+    .form-group
+      = f.label :sidekiq_throttling_queues, 'Sidekiq queues to throttle', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.select :sidekiq_throttling_queues, sidekiq_queue_options_for_select, { include_hidden: false }, multiple: true, class: 'select2 select-wide', data: { field: 'sidekiq_throttling_queues' }
+        .help-block
+          Choose which queues you wish to throttle.
+    .form-group
+      = f.label :sidekiq_throttling_factor, 'Throttling Factor', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :sidekiq_throttling_factor, class: 'form-control', min: '0', max: '0.99', step: '0.01'
+        .help-block
+          The factor by which the queues should be throttled. A value between 0.1 and 0.9.
 
   %fieldset
     %legend Spam and Anti-bot Protection
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index 6e660a8c026..7cc5e396f98 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -1,6 +1,3 @@
-require 'gitlab/current_settings'
-include Gitlab::CurrentSettings
-
 # Custom Redis configuration
 redis_config_hash = Gitlab::Redis.params
 redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
@@ -32,16 +29,11 @@ Sidekiq.configure_server do |config|
   end
   Sidekiq::Cron::Job.load_from_hash! cron_jobs
 
-  # allow it to fail: it may do so when create_from_defaults is executed before migrations are actually done
-  begin
-    throttling_enabled = current_application_settings.sidekiq_throttling_enabled
-  rescue
-    throttling_enabled = false
-  end
+  if Gitlab::CurrentSettings.sidekiq_throttling_enabled?
+    factor = current_application_settings.sidekiq_throttling_factor
 
-  if throttling_enabled
-    { 'project_cache' => 0.1, 'pipeline' => 0.1 }.each do |queue, ratio|
-      Sidekiq::Queue[queue].limit = (ratio * Sidekiq.options[:concurrency]).ceil
+    current_application_settings.sidekiq_throttling_queues.each do |queue|
+      Sidekiq::Queue[queue].limit = (factor * Sidekiq.options[:concurrency]).ceil
     end
   end
 
diff --git a/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb b/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb
index e2839219fb7..e644a174964 100644
--- a/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb
+++ b/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb
@@ -25,5 +25,7 @@ class AddSidekiqThrottlingToApplicationSettings < ActiveRecord::Migration
 
   def change
     add_column :application_settings, :sidekiq_throttling_enabled, :boolean, default: false
+    add_column :application_settings, :sidekiq_throttling_queues, :string
+    add_column :application_settings, :sidekiq_throttling_factor, :decimal
   end
 end
diff --git a/db/schema.rb b/db/schema.rb
index 31d01403508..666b54690c1 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -99,6 +99,8 @@ ActiveRecord::Schema.define(version: 20161106185620) do
     t.text "shared_runners_text_html"
     t.text "after_sign_up_text_html"
     t.boolean "sidekiq_throttling_enabled", default: false
+    t.string "sidekiq_throttling_queues"
+    t.decimal "sidekiq_throttling_factor"
     t.boolean "housekeeping_enabled", default: true, null: false
     t.boolean "housekeeping_bitmaps_enabled", default: true, null: false
     t.integer "housekeeping_incremental_repack_period", default: 10, null: false
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 801c2d5abcd..3a651ef318a 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -23,6 +23,10 @@ module Gitlab
       settings || fake_application_settings
     end
 
+    def sidekiq_throttling_enabled?
+      current_application_settings.sidekiq_throttling_enabled
+    end
+
     def fake_application_settings
       OpenStruct.new(
         default_projects_limit: Settings.gitlab['default_projects_limit'],