Merge branch 'master' into markdown-tags

Use the latest HTML pipeline gem

1 files changed, 53 insertions, 0 deletions

diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
new file mode 100644
index 00000000000..0f2db50e98c
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb
@@ -0,0 +1,53 @@
+module Gitlab
+  module SidekiqMiddleware
+    class MemoryKiller
+      # Default the RSS limit to 0, meaning the MemoryKiller is disabled
+      MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
+      # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
+      GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
+      # Wait 30 seconds for running jobs to finish during graceful shutdown
+      SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+
+      # Create a mutex used to ensure there will be only one thread waiting to
+      # shut Sidekiq down
+      MUTEX = Mutex.new
+
+      def call(worker, job, queue)
+        yield
+        current_rss = get_rss
+
+        return unless MAX_RSS > 0 && current_rss > MAX_RSS
+
+        Thread.new do
+          # Return if another thread is already waiting to shut Sidekiq down
+          return unless MUTEX.try_lock
+
+          Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
+            "#{MAX_RSS}"
+          Sidekiq.logger.warn "spawned thread that will shut down PID "\
+            "#{Process.pid} in #{GRACE_TIME} seconds"
+          sleep(GRACE_TIME)
+
+          Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
+          Process.kill('SIGUSR1', Process.pid)
+
+          Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\
+            "SIGTERM to PID #{Process.pid}"
+          sleep(SHUTDOWN_WAIT)
+
+          Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}"
+          Process.kill('SIGTERM', Process.pid)
+        end
+      end
+
+      private
+
+      def get_rss
+        output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid}))
+        return 0 unless status.zero?
+
+        output.to_i
+      end
+    end
+  end
+end