diff options
author | Jacob Vosmaer <contact@jacobvosmaer.nl> | 2015-05-07 18:47:03 +0200 |
---|---|---|
committer | Jacob Vosmaer <contact@jacobvosmaer.nl> | 2015-05-07 18:47:03 +0200 |
commit | 1c1f18b416febb8296facca37dc7504c2da880f8 (patch) | |
tree | 1db7a7015dcae8ba78cf133789bfccd6bd3c8d01 /lib/gitlab | |
parent | 66f7cdeb0af70100b0c60163c4685ba671413ddd (diff) | |
download | gitlab-ce-1c1f18b416febb8296facca37dc7504c2da880f8.tar.gz |
Add SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL env var
It looks like SIGTERM may not be enough to shut down a Sidekiq process
when its RSS has gotten too big. This change will allow us to experiment
with sending SIGKILL instead of SIGTERM to Sidekiq processes on
gitlab.com.
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/sidekiq_middleware/memory_killer.rb | 11 |
1 files changed, 6 insertions, 5 deletions
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 0f2db50e98c..f33b2dedf4a 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -7,6 +7,7 @@ module Gitlab GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i # Wait 30 seconds for running jobs to finish during graceful shutdown SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + SHUTDOWN_SIGNAL = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL'] || 'SIGTERM').to_s # Create a mutex used to ensure there will be only one thread waiting to # shut Sidekiq down @@ -24,19 +25,19 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "spawned thread that will shut down PID "\ - "#{Process.pid} in #{GRACE_TIME} seconds" + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\ + "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" Process.kill('SIGUSR1', Process.pid) Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ - "SIGTERM to PID #{Process.pid}" + "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}" sleep(SHUTDOWN_WAIT) - Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" - Process.kill('SIGTERM', Process.pid) + Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + Process.kill(SHUTDOWN_SIGNAL, Process.pid) end end |