Merge branch '56850-add-new-unicorn-metrics' into 'master'

Resolve "Add new Unicorn metrics"

Closes #56850

See merge request gitlab-org/gitlab-ce!27474

3 files changed, 69 insertions, 17 deletions

diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index 18a69321905..4d9c43f37e7 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -23,25 +23,32 @@ module Gitlab
         end
 
         def init_metrics
-          metrics = {}
-          metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels)
-          metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
+          metrics = {
+            file_descriptors:               ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum),
+            memory_bytes:                   ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum),
+            process_cpu_seconds_total:      ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'),
+            process_max_fds:                ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'),
+            process_resident_memory_bytes:  ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum),
+            process_start_time_seconds:     ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
+            sampler_duration:               ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
+            total_time:                     ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
+          }
+
           GC.stat.keys.each do |key|
             metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum)
           end
 
-          metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum)
-          metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum)
-
           metrics
         end
 
         def sample
           start_time = System.monotonic_time
 
-          metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage)
           metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count)
-
+          metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time)
+          metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors)
+          metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time)
+          set_memory_usage_metrics
           sample_gc
 
           metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time)
@@ -61,6 +68,14 @@ module Gitlab
           metrics[:total_time].increment(labels, GC::Profiler.total_time)
         end
 
+        def set_memory_usage_metrics
+          memory_usage = System.memory_usage
+          memory_labels = labels.merge(worker_label)
+
+          metrics[:memory_bytes].set(memory_labels, memory_usage)
+          metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage)
+        end
+
         def worker_label
           return {} unless defined?(Unicorn::Worker)
 
diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
index bec64e864b3..1b6c52ac0bf 100644
--- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb
+++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
@@ -8,12 +8,16 @@ module Gitlab
           super(interval)
         end
 
-        def unicorn_active_connections
-          @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max)
+        def metrics
+          @metrics ||= init_metrics
         end
 
-        def unicorn_queued_connections
-          @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max)
+        def init_metrics
+          {
+            unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max),
+            unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max),
+            unicorn_workers:            ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers')
+          }
         end
 
         def enabled?
@@ -23,14 +27,13 @@ module Gitlab
 
         def sample
           Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats|
-            unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active)
-            unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued)
+            set_unicorn_connection_metrics('tcp', addr, stats)
           end
-
           Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats|
-            unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active)
-            unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued)
+            set_unicorn_connection_metrics('unix', addr, stats)
           end
+
+          metrics[:unicorn_workers].set({}, unicorn_workers_count)
         end
 
         private
@@ -39,6 +42,13 @@ module Gitlab
           @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z})
         end
 
+        def set_unicorn_connection_metrics(type, addr, stats)
+          labels = { socket_type: type, socket_address: addr }
+
+          metrics[:unicorn_active_connections].set(labels, stats.active)
+          metrics[:unicorn_queued_connections].set(labels, stats.queued)
+        end
+
         def unix_listeners
           @unix_listeners ||= Unicorn.listener_names - tcp_listeners
         end
@@ -46,6 +56,10 @@ module Gitlab
         def unicorn_with_listeners?
           defined?(Unicorn) && Unicorn.listener_names.any?
         end
+
+        def unicorn_workers_count
+          Sys::ProcTable.ps.select {|p| p.cmdline.match(/unicorn_rails worker.+ #{Rails.root.to_s}/)}.count
+        end
       end
     end
   end
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index 426496855e3..ce0bb82d138 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -23,6 +23,21 @@ module Gitlab
         def self.file_descriptor_count
           Dir.glob('/proc/self/fd/*').length
         end
+
+        def self.max_open_file_descriptors
+          match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/)
+
+          return unless match && match[1]
+
+          match[1].to_i
+        end
+
+        def self.process_start_time
+          start_time_in_jiffies = Sys::ProcTable.ps(pid: Process.pid).starttime
+          return 0 unless start_time_in_jiffies
+
+          start_time_in_jiffies / 100
+        end
       else
         def self.memory_usage
           0.0
@@ -31,6 +46,14 @@ module Gitlab
         def self.file_descriptor_count
           0
         end
+
+        def self.max_open_file_descriptors
+          0
+        end
+
+        def self.process_start_time
+          0
+        end
       end
 
       # THREAD_CPUTIME is not supported on OS X