Merge branch 'puma-mem-counter' into 'master'

Add puma killer observer

Closes #63037

See merge request gitlab-org/gitlab-ce!28992

6 files changed, 56 insertions, 2 deletions

diff --git a/config/puma.example.development.rb b/config/puma.example.development.rb
index 490c940077a..9df24bf74e3 100644
--- a/config/puma.example.development.rb
+++ b/config/puma.example.development.rb
@@ -42,7 +42,6 @@ bind 'unix:///home/git/gitlab.socket'
 workers 2
 
 require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
-require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer"
 
 on_restart do
   # Signal application hooks that we're about to restart
diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md
index 3dcd1593099..84b71ae6f1c 100644
--- a/doc/administration/monitoring/prometheus/gitlab_metrics.md
+++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md
@@ -125,6 +125,7 @@ When Puma is used instead of Unicorn, following metrics are available:
 | puma_max_threads                             | Gauge   | 12.0  | Maximum number of worker threads |
 | puma_idle_threads                            | Gauge   | 12.0  | Number of spawned threads which are not processing a request |
 | rack_state_total                             | Gauge   | 12.0  | Number of requests in a given rack state |
+| puma_killer_terminations_total               | Gauge   | 12.0  | Number of workers terminated by PumaWorkerKiller |
 
 ## Metrics shared directory
 
diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
index 4ed9a9a02ab..4affc52b7b0 100644
--- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb
+++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
@@ -27,6 +27,9 @@ module Gitlab
           # is restarted already, thus periodically restarting workers shouldn't be
           # needed.
           config.rolling_restart_frequency = false
+
+          observer = Gitlab::Cluster::PumaWorkerKillerObserver.new
+          config.pre_term = observer.callback
         end
 
         PumaWorkerKiller.start
diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb
new file mode 100644
index 00000000000..3b4ebc3fbae
--- /dev/null
+++ b/lib/gitlab/cluster/puma_worker_killer_observer.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cluster
+    class PumaWorkerKillerObserver
+      def initialize
+        @counter = Gitlab::Metrics.counter(:puma_killer_terminations_total, 'Number of workers terminated by PumaWorkerKiller')
+      end
+
+      # returns the Proc to be used as the observer callback block
+      def callback
+        method(:log_termination)
+      end
+
+      private
+
+      def log_termination(worker)
+        labels = { worker: "worker_#{worker.index}" }
+
+        @counter.increment(labels)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb b/spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb
new file mode 100644
index 00000000000..180520b27e7
--- /dev/null
+++ b/spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Cluster::PumaWorkerKillerObserver do
+  let(:counter) { Gitlab::Metrics::NullMetric.instance }
+
+  before do
+    allow(Gitlab::Metrics).to receive(:counter)
+      .with(any_args)
+      .and_return(counter)
+  end
+
+  describe '#callback' do
+    subject { described_class.new }
+
+    it 'increments timeout counter' do
+      worker = double(index: 0)
+
+      expect(counter)
+        .to receive(:increment)
+        .with({ worker: 'worker_0' })
+
+      subject.callback.call(worker)
+    end
+  end
+end
diff --git a/spec/rack_servers/puma_spec.rb b/spec/rack_servers/puma_spec.rb
index 8290473821c..a4b37905af3 100644
--- a/spec/rack_servers/puma_spec.rb
+++ b/spec/rack_servers/puma_spec.rb
@@ -20,7 +20,7 @@ describe 'Puma' do
     File.write(config_path, config_lines)
 
     cmd = %W[puma -e test -C #{config_path} #{File.join(__dir__, 'configs/config.ru')}]
-    @puma_master_pid = spawn(*cmd)
+    @puma_master_pid = spawn({ 'DISABLE_PUMA_WORKER_KILLER' => '1' }, *cmd)
     wait_puma_boot!(@puma_master_pid, File.join(project_root, 'tmp/tests/puma-worker-ready'))
     WebMock.allow_net_connect!
   end