diff options
author | Stan Hu <stanhu@gmail.com> | 2019-07-25 21:32:39 +0000 |
---|---|---|
committer | Stan Hu <stanhu@gmail.com> | 2019-07-25 21:32:39 +0000 |
commit | 3736e1de27ab05a280cc78e6542b4e41f94aa805 (patch) | |
tree | dfcd411f587520175b51a3918ac0b1f762348ece | |
parent | 57aabe16b73890d798ab86dccb3a845aa25d5232 (diff) | |
parent | ad7acfaad0e806cac48889451631061775a6dfa2 (diff) | |
download | gitlab-ce-3736e1de27ab05a280cc78e6542b4e41f94aa805.tar.gz |
Merge branch 'print-sidekiq-class-name-in-memory-killer' into 'master'
Logging sidekiq worker class name in SidekiqMemoryKiller
See merge request gitlab-org/gitlab-ce!30996
-rw-r--r-- | lib/gitlab/sidekiq_middleware/memory_killer.rb | 23 | ||||
-rw-r--r-- | spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb | 3 |
2 files changed, 19 insertions, 7 deletions
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 671d795ec33..4b10f921ed8 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -14,9 +14,12 @@ module Gitlab # shut Sidekiq down MUTEX = Mutex.new + attr_reader :worker + def call(worker, job, queue) yield + @worker = worker current_rss = get_rss return unless MAX_RSS > 0 && current_rss > MAX_RSS @@ -25,9 +28,11 @@ module Gitlab # Return if another thread is already waiting to shut Sidekiq down next unless MUTEX.try_lock - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") + + warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ + " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") # Wait `GRACE_TIME` to give the memory intensive job time to finish. # Then, tell Sidekiq to stop fetching new jobs. @@ -59,24 +64,28 @@ module Gitlab def wait_and_signal_pgroup(time, signal, explanation) return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, 0) end def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, pid) end def pid Process.pid end + + def warn(message, signal: nil) + Sidekiq.logger.warn(class: worker.class, pid: pid, signal: signal, message: message) + end end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb index 1a5a38b5d99..b451844f06c 100644 --- a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb @@ -45,6 +45,9 @@ describe Gitlab::SidekiqMiddleware::MemoryKiller do expect(subject).to receive(:sleep).with(10).ordered expect(Process).to receive(:kill).with('SIGKILL', pid).ordered + expect(Sidekiq.logger) + .to receive(:warn).with(class: 'TestWorker', message: anything, pid: pid, signal: anything).at_least(:once) + run end |