diff options
| author | James Edwards-Jones <jedwardsjones@gitlab.com> | 2018-06-20 13:48:03 +1000 |
|---|---|---|
| committer | James Edwards-Jones <jedwardsjones@gitlab.com> | 2018-06-20 13:48:03 +1000 |
| commit | 7b155840128fa689fd222abf164c9e603b07be5e (patch) | |
| tree | 2bd8825bfb25da5b289c8b681d70551c3c3a838a | |
| parent | da11d3ca868abaa496ddc398a82494cd73b284bf (diff) | |
| download | gitlab-ce-jej/sidekiq-subprocess-killer.tar.gz | |
WIP: manual testingjej/sidekiq-subprocess-killer
| -rw-r--r-- | app/workers/all_queues.yml | 1 | ||||
| -rw-r--r-- | app/workers/slow_yes_worker.rb | 10 | ||||
| -rw-r--r-- | config/sidekiq_queues.yml | 1 | ||||
| -rw-r--r-- | lib/gitlab/sidekiq_middleware/shutdown.rb | 16 |
4 files changed, 26 insertions, 2 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index c469aea7052..2c46c560356 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -112,3 +112,4 @@ - update_user_activity - upload_checksum - web_hook +- slow_yes diff --git a/app/workers/slow_yes_worker.rb b/app/workers/slow_yes_worker.rb new file mode 100644 index 00000000000..f1149332f76 --- /dev/null +++ b/app/workers/slow_yes_worker.rb @@ -0,0 +1,10 @@ +class SlowYesWorker + include ApplicationWorker + + def perform + system('unzip -o /tmp/large.zip') + rescue => e + Gitlab::AppLogger.error(e.message) + raise e + end +end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 47fbbed44cf..9f069e60ef0 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -73,3 +73,4 @@ - [object_storage, 1] - [plugin, 1] - [pipeline_background, 1] + - [slow_yes, 1] diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb index 5546fba0ca6..b6d19e63279 100644 --- a/lib/gitlab/sidekiq_middleware/shutdown.rb +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -8,9 +8,11 @@ module Gitlab # Default the RSS limit to 0, meaning the MemoryKiller is disabled MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + GRACE_TIME = 2 # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + # SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + SHUTDOWN_WAIT = 5 # Wait additional time for Sidekiq to finish terminatring # and for subprocesses to terminate ADDITIONAL_WAIT = 2 @@ -52,6 +54,7 @@ module Gitlab shutdown_exception = nil begin + check_manual_shutdown! yield check_rss! rescue WantShutdown => ex @@ -69,6 +72,15 @@ module Gitlab private + # This is a temporary method for reproducing Shutdown + def check_manual_shutdown! + return unless File.exists?('/tmp/shutdown.sidekiq') + + File.delete('/tmp/shutdown.sidekiq') + + raise ShutdownWithoutRaise.new('Shutdown initiated by /tmp/shutdown.sidekiq') + end + def do_shutdown(worker, job, shutdown_exception) Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ "#{worker.class} JID-#{job['jid']}" |