From 7b155840128fa689fd222abf164c9e603b07be5e Mon Sep 17 00:00:00 2001
From: James Edwards-Jones <jedwardsjones@gitlab.com>
Date: Wed, 20 Jun 2018 13:48:03 +1000
Subject: WIP: manual testing

---
 app/workers/all_queues.yml                |  1 +
 app/workers/slow_yes_worker.rb            | 10 ++++++++++
 config/sidekiq_queues.yml                 |  1 +
 lib/gitlab/sidekiq_middleware/shutdown.rb | 16 ++++++++++++++--
 4 files changed, 26 insertions(+), 2 deletions(-)
 create mode 100644 app/workers/slow_yes_worker.rb

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index c469aea7052..2c46c560356 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -112,3 +112,4 @@
 - update_user_activity
 - upload_checksum
 - web_hook
+- slow_yes
diff --git a/app/workers/slow_yes_worker.rb b/app/workers/slow_yes_worker.rb
new file mode 100644
index 00000000000..f1149332f76
--- /dev/null
+++ b/app/workers/slow_yes_worker.rb
@@ -0,0 +1,10 @@
+class SlowYesWorker
+  include ApplicationWorker
+
+  def perform
+    system('unzip -o /tmp/large.zip')
+  rescue => e
+    Gitlab::AppLogger.error(e.message)
+    raise e
+  end
+end
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 47fbbed44cf..9f069e60ef0 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -73,3 +73,4 @@
   - [object_storage, 1]
   - [plugin, 1]
   - [pipeline_background, 1]
+  - [slow_yes, 1]
diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb
index 5546fba0ca6..b6d19e63279 100644
--- a/lib/gitlab/sidekiq_middleware/shutdown.rb
+++ b/lib/gitlab/sidekiq_middleware/shutdown.rb
@@ -8,9 +8,11 @@ module Gitlab
       # Default the RSS limit to 0, meaning the MemoryKiller is disabled
       MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
       # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
-      GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
+      # GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
+      GRACE_TIME = 2
       # Wait 30 seconds for running jobs to finish during graceful shutdown
-      SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+      # SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+      SHUTDOWN_WAIT = 5
       # Wait additional time for Sidekiq to finish terminatring
       # and for subprocesses to terminate
       ADDITIONAL_WAIT = 2
@@ -52,6 +54,7 @@ module Gitlab
         shutdown_exception = nil
 
         begin
+          check_manual_shutdown!
           yield
           check_rss!
         rescue WantShutdown => ex
@@ -69,6 +72,15 @@ module Gitlab
 
       private
 
+      # This is a temporary method for reproducing Shutdown
+      def check_manual_shutdown!
+        return unless File.exists?('/tmp/shutdown.sidekiq')
+
+        File.delete('/tmp/shutdown.sidekiq')
+          
+        raise ShutdownWithoutRaise.new('Shutdown initiated by /tmp/shutdown.sidekiq')
+      end
+
       def do_shutdown(worker, job, shutdown_exception)
         Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\
           "#{worker.class} JID-#{job['jid']}"
-- 
cgit v1.2.1