Add SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL env var

It looks like SIGTERM may not be enough to shut down a Sidekiq process
when its RSS has gotten too big. This change will allow us to experiment
with sending SIGKILL instead of SIGTERM to Sidekiq processes on
gitlab.com.

1 files changed, 6 insertions, 5 deletions

diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
index 0f2db50e98c..f33b2dedf4a 100644
--- a/lib/gitlab/sidekiq_middleware/memory_killer.rb
+++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb
@@ -7,6 +7,7 @@ module Gitlab
       GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
       # Wait 30 seconds for running jobs to finish during graceful shutdown
       SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+      SHUTDOWN_SIGNAL = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL'] || 'SIGTERM').to_s
 
       # Create a mutex used to ensure there will be only one thread waiting to
       # shut Sidekiq down
@@ -24,19 +25,19 @@ module Gitlab
 
           Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
             "#{MAX_RSS}"
-          Sidekiq.logger.warn "spawned thread that will shut down PID "\
-            "#{Process.pid} in #{GRACE_TIME} seconds"
+          Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\
+            "in #{GRACE_TIME} seconds"
           sleep(GRACE_TIME)
 
           Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
           Process.kill('SIGUSR1', Process.pid)
 
           Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\
-            "SIGTERM to PID #{Process.pid}"
+            "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
           sleep(SHUTDOWN_WAIT)
 
-          Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}"
-          Process.kill('SIGTERM', Process.pid)
+          Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
+          Process.kill(SHUTDOWN_SIGNAL, Process.pid)
         end
       end