diff options
| -rw-r--r-- | Gemfile | 1 | ||||
| -rw-r--r-- | Gemfile.lock | 3 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/unicorn_sampler.rb | 36 | ||||
| -rw-r--r-- | lib/gitlab/metrics/system.rb | 25 | ||||
| -rw-r--r-- | spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb | 33 | ||||
| -rw-r--r-- | spec/lib/gitlab/metrics/system_spec.rb | 18 |
6 files changed, 103 insertions, 13 deletions
@@ -406,6 +406,7 @@ gem 'health_check', '~> 2.6.0' # System information gem 'vmstat', '~> 2.3.0' gem 'sys-filesystem', '~> 1.1.6' +gem 'sys-proctable', '~> 1.2' # SSH host key support gem 'net-ssh', '~> 5.0' diff --git a/Gemfile.lock b/Gemfile.lock index 109958e2591..36c24265e48 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -867,6 +867,8 @@ GEM state_machines-activemodel (>= 0.5.0) sys-filesystem (1.1.6) ffi + sys-proctable (1.2.1) + ffi sysexits (1.2.0) temple (0.8.0) test-prof (0.2.5) @@ -1160,6 +1162,7 @@ DEPENDENCIES stackprof (~> 0.2.10) state_machines-activerecord (~> 0.5.1) sys-filesystem (~> 1.1.6) + sys-proctable (~> 1.2) test-prof (~> 0.2.5) thin (~> 1.7.0) timecop (~> 0.8.0) diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index bec64e864b3..16a2ee9b9be 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -8,12 +8,19 @@ module Gitlab super(interval) end - def unicorn_active_connections - @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + def metrics + @metrics ||= init_metrics end - def unicorn_queued_connections - @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + def init_metrics + { + unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), + unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), + unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers'), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'), + process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds') + } end def enabled? @@ -23,14 +30,19 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) + metrics[:unicorn_active_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.active) + metrics[:unicorn_queued_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.queued) end Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) + metrics[:unicorn_active_connections].set({ socket_type: 'unix', socket_address: addr }, stats.active) + metrics[:unicorn_queued_connections].set({ socket_type: 'unix', socket_address: addr }, stats.queued) end + + metrics[:process_cpu_seconds_total].set({ pid: nil }, ::Gitlab::Metrics::System.cpu_time) + metrics[:process_start_time_seconds].set({ pid: nil }, ::Gitlab::Metrics::System.process_start_time) + metrics[:process_max_fds].set({ pid: nil }, ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:unicorn_workers].set({}, unicorn_workers_count) end private @@ -39,6 +51,10 @@ module Gitlab @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) end + def pid + @pid ||= Process.pid + end + def unix_listeners @unix_listeners ||= Unicorn.listener_names - tcp_listeners end @@ -46,6 +62,10 @@ module Gitlab def unicorn_with_listeners? defined?(Unicorn) && Unicorn.listener_names.any? end + + def unicorn_workers_count + Sys::ProcTable.ps.select {|p| p.cmdline.match(/unicorn_rails worker/)}.count + end end end end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 426496855e3..a269a8688e9 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -23,6 +23,16 @@ module Gitlab def self.file_descriptor_count Dir.glob('/proc/self/fd/*').length end + + def self.max_open_file_descriptors + match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/) + + if match && match[1] + max_fds = match[1].to_i + end + + max_fds + end else def self.memory_usage 0.0 @@ -31,6 +41,10 @@ module Gitlab def self.file_descriptor_count 0 end + + def self.max_open_file_descriptors + 0 + end end # THREAD_CPUTIME is not supported on OS X @@ -46,6 +60,17 @@ module Gitlab end end + # CLOCK_BOOTTIME is not supported on OS X + if Process.const_defined?(:CLOCK_BOOTTIME) + def self.process_start_time + Process + .clock_gettime(Process::CLOCK_BOOTTIME, :float_second) + end + else + def self.process_start_time + 0.0 + end + end # Returns the current real time in a given precision. # # Returns the time as a Float for precision = :float_second. diff --git a/spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb index 4b03f3c2532..4470dc3ee93 100644 --- a/spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb +++ b/spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb @@ -39,8 +39,8 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do it 'updates metrics type unix and with addr' do labels = { socket_type: 'unix', socket_address: socket_address } - expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active') - expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued') + expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active') + expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued') subject.sample end @@ -50,7 +50,6 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do context 'unicorn listens on tcp sockets' do let(:tcp_socket_address) { '0.0.0.0:8080' } let(:tcp_sockets) { [tcp_socket_address] } - before do allow(unicorn).to receive(:listener_names).and_return(tcp_sockets) end @@ -71,13 +70,37 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do it 'updates metrics type unix and with addr' do labels = { socket_type: 'tcp', socket_address: tcp_socket_address } - expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active') - expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued') + expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active') + expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued') subject.sample end end end + + context 'additional metrics' do + let(:cpu_time) { 3.14 } + let(:process_start_time) { 19100.24 } + let(:process_max_fds) { 1024 } + let(:unicorn_workers) { 2 } + + before do + allow(unicorn).to receive(:listener_names).and_return([""]) + allow(::Gitlab::Metrics::System).to receive(:cpu_time).and_return(cpu_time) + allow(::Gitlab::Metrics::System).to receive(:process_start_time).and_return(process_start_time) + allow(::Gitlab::Metrics::System).to receive(:max_open_file_descriptors).and_return(process_max_fds) + allow(subject).to receive(:unicorn_workers_count).and_return(unicorn_workers) + end + + it "sets additional metrics" do + expect(subject.metrics[:process_cpu_seconds_total]).to receive(:set).with({ pid: nil }, cpu_time) + expect(subject.metrics[:process_start_time_seconds]).to receive(:set).with({ pid: nil }, process_start_time) + expect(subject.metrics[:process_max_fds]).to receive(:set).with({ pid: nil }, process_max_fds) + expect(subject.metrics[:unicorn_workers]).to receive(:set).with({}, unicorn_workers) + + subject.sample + end + end end describe '#start' do diff --git a/spec/lib/gitlab/metrics/system_spec.rb b/spec/lib/gitlab/metrics/system_spec.rb index 14afcdf5daa..2de6821bb79 100644 --- a/spec/lib/gitlab/metrics/system_spec.rb +++ b/spec/lib/gitlab/metrics/system_spec.rb @@ -13,6 +13,12 @@ describe Gitlab::Metrics::System do expect(described_class.file_descriptor_count).to be > 0 end end + + describe '.max_open_file_descriptors' do + it 'returns the max allowed open file descriptors' do + expect(described_class.max_open_file_descriptors).to be > 0 + end + end else describe '.memory_usage' do it 'returns 0.0' do @@ -25,6 +31,12 @@ describe Gitlab::Metrics::System do expect(described_class.file_descriptor_count).to eq(0) end end + + describe '.max_open_file_descriptors' do + it 'returns 0' do + expect(described_class.max_open_file_descriptors).to eq(0) + end + end end describe '.cpu_time' do @@ -44,4 +56,10 @@ describe Gitlab::Metrics::System do expect(described_class.monotonic_time).to be_an(Float) end end + + describe '.process_start_time' do + it 'returns a Float' do + expect(described_class.process_start_time).to be_an(Float) + end + end end |