diff options
author | Jacob Vosmaer <contact@jacobvosmaer.nl> | 2014-12-05 17:29:34 +0100 |
---|---|---|
committer | Jacob Vosmaer <contact@jacobvosmaer.nl> | 2014-12-05 17:29:34 +0100 |
commit | 4f9a14061b707f39d8a98dae328089bbfbc09e70 (patch) | |
tree | e0fdcbf816718db52af406f5a96569fd5ac77001 /lib | |
parent | eb12e3ab56eecb05ea2847b414554266cfb2947c (diff) | |
download | gitlab-ce-4f9a14061b707f39d8a98dae328089bbfbc09e70.tar.gz |
Wait 15 minutes before Sidekiq MemoryKiller action
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/sidekiq_middleware/memory_killer.rb | 48 |
1 files changed, 33 insertions, 15 deletions
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 0fb09d3f228..df8968cf677 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -1,26 +1,38 @@ module Gitlab module SidekiqMiddleware class MemoryKiller + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = 15 * 60 # Wait 30 seconds for running jobs to finish during graceful shutdown - GRACEFUL_SHUTDOWN_WAIT = 30 + SHUTDOWN_WAIT = 30 + # Create a mutex so that there will be only one thread waiting to shut + # Sidekiq down + MUTEX = Mutex.new def call(worker, job, queue) yield current_rss = get_rss + return unless max_rss > 0 && current_rss > max_rss - Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ - "#{max_rss}" - Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" - # SIGUSR1 tells Sidekiq to stop accepting new jobs - Process.kill('SIGUSR1', Process.pid) - - Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\ - "#{Process.pid} in #{graceful_shutdown_wait} seconds" - # Send the final shutdown signal to Sidekiq from a separate thread so - # that the current job can finish - Thread.new do - sleep(graceful_shutdown_wait) + Tread.new do + # Return if another thread is already waiting to shut Sidekiq down + return unless MUTEX.try_lock + + Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ + "#{max_rss}" + Sidekiq.logger.warn "spawned thread that will shut down PID "\ + "#{Process.pid} in #{grace_time} seconds" + sleep(grace_time) + + Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + Process.kill('SIGUSR1', Process.pid) + + Sidekiq.logger.warn "waiting #{shutdown_wait} seconds before sending "\ + "SIGTERM to PID #{Process.pid}" + sleep(shutdown_wait) + + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" Process.kill('SIGTERM', Process.pid) end end @@ -38,9 +50,15 @@ module Gitlab @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i end - def graceful_shutdown_wait + def shutdown_wait @graceful_shutdown_wait ||= ( - ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT + ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || SHUTDOWN_WAIT + ).to_i + end + + def grace_time + @grace_time ||= ( + ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || GRACE_TIME ).to_i end end |