Adds ruby and unicorn instrumentation

This adds ruby and unicorn instrumentation. This was originally
intended in 11.11 but due to performance concerns it was reverted. This
new commit foregoes the sys-proctable gem was causing performance issues
previously.

3 files changed, 72 insertions, 17 deletions

diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index 18a69321905..4d9c43f37e7 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -23,25 +23,32 @@ module Gitlab
         end
 
         def init_metrics
-          metrics = {}
-          metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels)
-          metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
+          metrics = {
+            file_descriptors:               ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum),
+            memory_bytes:                   ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum),
+            process_cpu_seconds_total:      ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'),
+            process_max_fds:                ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'),
+            process_resident_memory_bytes:  ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum),
+            process_start_time_seconds:     ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
+            sampler_duration:               ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
+            total_time:                     ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
+          }
+
           GC.stat.keys.each do |key|
             metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum)
           end
 
-          metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum)
-          metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum)
-
           metrics
         end
 
         def sample
           start_time = System.monotonic_time
 
-          metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage)
           metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count)
-
+          metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time)
+          metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors)
+          metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time)
+          set_memory_usage_metrics
           sample_gc
 
           metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time)
@@ -61,6 +68,14 @@ module Gitlab
           metrics[:total_time].increment(labels, GC::Profiler.total_time)
         end
 
+        def set_memory_usage_metrics
+          memory_usage = System.memory_usage
+          memory_labels = labels.merge(worker_label)
+
+          metrics[:memory_bytes].set(memory_labels, memory_usage)
+          metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage)
+        end
+
         def worker_label
           return {} unless defined?(Unicorn::Worker)
 
diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
index bec64e864b3..1b7a4c79080 100644
--- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb
+++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
@@ -8,12 +8,16 @@ module Gitlab
           super(interval)
         end
 
-        def unicorn_active_connections
-          @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max)
+        def metrics
+          @metrics ||= init_metrics
         end
 
-        def unicorn_queued_connections
-          @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max)
+        def init_metrics
+          {
+            unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max),
+            unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max),
+            unicorn_workers:            ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers')
+          }
         end
 
         def enabled?
@@ -23,14 +27,13 @@ module Gitlab
 
         def sample
           Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats|
-            unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active)
-            unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued)
+            set_unicorn_connection_metrics('tcp', addr, stats)
           end
-
           Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats|
-            unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active)
-            unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued)
+            set_unicorn_connection_metrics('unix', addr, stats)
           end
+
+          metrics[:unicorn_workers].set({}, unicorn_workers_count)
         end
 
         private
@@ -39,6 +42,13 @@ module Gitlab
           @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z})
         end
 
+        def set_unicorn_connection_metrics(type, addr, stats)
+          labels = { socket_type: type, socket_address: addr }
+
+          metrics[:unicorn_active_connections].set(labels, stats.active)
+          metrics[:unicorn_queued_connections].set(labels, stats.queued)
+        end
+
         def unix_listeners
           @unix_listeners ||= Unicorn.listener_names - tcp_listeners
         end
@@ -46,6 +56,10 @@ module Gitlab
         def unicorn_with_listeners?
           defined?(Unicorn) && Unicorn.listener_names.any?
         end
+
+        def unicorn_workers_count
+          `pgrep -f '[u]nicorn_rails worker.+ #{Rails.root.to_s}'`.split.count
+        end
       end
     end
   end
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index 426496855e3..d3cbd200584 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -23,6 +23,20 @@ module Gitlab
         def self.file_descriptor_count
           Dir.glob('/proc/self/fd/*').length
         end
+
+        def self.max_open_file_descriptors
+          match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/)
+
+          return unless match && match[1]
+
+          match[1].to_i
+        end
+
+        def self.process_start_time
+          fields = File.read('/proc/self/stat').split
+
+          ( fields[21].to_i || 0 ) / clk_tck
+        end
       else
         def self.memory_usage
           0.0
@@ -31,6 +45,14 @@ module Gitlab
         def self.file_descriptor_count
           0
         end
+
+        def self.max_open_file_descriptors
+          0
+        end
+
+        def self.process_start_time
+          0
+        end
       end
 
       # THREAD_CPUTIME is not supported on OS X
@@ -59,6 +81,10 @@ module Gitlab
       def self.monotonic_time
         Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
       end
+
+      def self.clk_tck
+        @clk_tck ||= `genconf CLK_TCK`.to_i
+      end
     end
   end
 end