diff options
author | Jacob Vosmaer <contact@jacobvosmaer.nl> | 2014-11-28 15:01:41 +0100 |
---|---|---|
committer | Jacob Vosmaer <contact@jacobvosmaer.nl> | 2014-11-28 15:01:41 +0100 |
commit | 64ab6c9ed54d1c0a86f4c3bb6b87fcac882da0c0 (patch) | |
tree | acb68d37d12268d43cf87a89fe8094c744268685 /lib/gitlab/sidekiq_middleware | |
parent | 3a723ad24c56404a4180b011bd577153f9d25ccd (diff) | |
download | gitlab-ce-64ab6c9ed54d1c0a86f4c3bb6b87fcac882da0c0.tar.gz |
Add 'MemoryKiller' Sidekiq middleware
When enabled, this middleware allows Sidekiq to detect that its RSS has
exceeded a maximum value, triggering a graceful shutdown. This
middleware should be combined with external process supervision that
will restart Sidekiq after the graceful shutdown, such as Runit.
Diffstat (limited to 'lib/gitlab/sidekiq_middleware')
-rw-r--r-- | lib/gitlab/sidekiq_middleware/memory_killer.rb | 45 |
1 files changed, 45 insertions, 0 deletions
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb new file mode 100644 index 00000000000..3ef46627916 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -0,0 +1,45 @@ +module Gitlab + module SidekiqMiddleware + class MemoryKiller + # Wait 30 seconds for running jobs to finish during graceful shutdown + GRACEFUL_SHUTDOWN_WAIT = 30 + + def call(worker, job, queue) + yield + current_rss = get_rss + return unless max_rss > 0 && current_rss > max_rss + + Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ + "#{max_rss}" + Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + Process.kill('SIGUSR1', Process.pid) + + Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\ + "#{Process.pid} in #{graceful_shutdown_wait} seconds" + Thread.new do + sleep(graceful_shutdown_wait) + Process.kill('SIGTERM', Process.pid) + end + end + + private + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid})) + return 0 unless status.zero? + + output.to_i + end + + def max_rss + @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i + end + + def graceful_shutdown_wait + @graceful_shutdown_wait ||= ( + ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT + ).to_i + end + end + end +end |