diff options
| -rw-r--r-- | app/controllers/health_controller.rb | 4 | ||||
| -rw-r--r-- | app/services/prom_service.rb | 1 | ||||
| -rw-r--r-- | config.ru | 6 | ||||
| -rw-r--r-- | lib/gitlab/metrics.rb | 65 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dummy_metric.rb | 29 | ||||
| -rw-r--r-- | lib/gitlab/metrics/prometheus_sampler.rb | 51 |
6 files changed, 146 insertions, 10 deletions
diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 125746d0426..03cc896fce9 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -1,3 +1,5 @@ +require 'prometheus/client/formats/text' + class HealthController < ActionController::Base protect_from_forgery with: :exception include RequiresHealthToken @@ -24,7 +26,7 @@ class HealthController < ActionController::Base results = CHECKS.flat_map(&:metrics) response = results.map(&method(:metric_to_prom_line)).join("\n") - + response = ::Prometheus::Client::Formats::Text.marshal_multiprocess render text: response, content_type: 'text/plain; version=0.0.4' end diff --git a/app/services/prom_service.rb b/app/services/prom_service.rb index e9ccce758e9..93d52fd2c79 100644 --- a/app/services/prom_service.rb +++ b/app/services/prom_service.rb @@ -11,6 +11,5 @@ class PromService @login = Prometheus::Client::Counter.new(:login, 'Login counter') @prometheus.register(@login) - end end diff --git a/config.ru b/config.ru index 82af814341b..cba44122da9 100644 --- a/config.ru +++ b/config.ru @@ -15,11 +15,7 @@ if defined?(Unicorn) end # TODO(lyda): Needs to be set externally. - ENV['prometheus_multiproc_dir'] = '/tmp' - - require 'prometheus/client/rack/exporter' - - use Prometheus::Client::Rack::Exporter, path: '/admin/metrics' + ENV['prometheus_multiproc_dir'] = '/tmp/somestuff' end require ::File.expand_path('../config/environment', __FILE__) diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cb8db2f1e9f..e784ca785f0 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,3 +1,5 @@ +require 'prometheus/client' + module Gitlab module Metrics extend Gitlab::CurrentSettings @@ -9,6 +11,7 @@ module Gitlab def self.settings @settings ||= { enabled: current_application_settings[:metrics_enabled], + prometheus_metrics_enabled: true, pool_size: current_application_settings[:metrics_pool_size], timeout: current_application_settings[:metrics_timeout], method_call_threshold: current_application_settings[:metrics_method_call_threshold], @@ -19,10 +22,18 @@ module Gitlab } end - def self.enabled? + def self.prometheus_metrics_enabled? + settings[:prometheus_metrics_enabled] || false + end + + def self.influx_metrics_enabled? settings[:enabled] || false end + def self.enabled? + influx_metrics_enabled? || prometheus_metrics_enabled? || false + end + def self.mri? RUBY_ENGINE == 'ruby' end @@ -38,10 +49,58 @@ module Gitlab @pool end + def self.registry + @registry ||= ::Prometheus::Client.registry + end + + def self.counter(name, docstring, base_labels = {}) + dummy_metric || registry.get(name) || registry.counter(name, docstring, base_labels) + end + + def self.summary(name, docstring, base_labels = {}) + dummy_metric || registry.get(name) || registry.summary(name, docstring, base_labels) + end + + def self.gauge(name, docstring, base_labels = {}) + dummy_metric || registry.get(name) || registry.gauge(name, docstring, base_labels) + end + + def self.histogram(name, docstring, base_labels = {}, buckets = Histogram::DEFAULT_BUCKETS) + dummy_metric || registry.get(name) || registry.histogram(name, docstring, base_labels, buckets) + end + + def self.dummy_metric + unless prometheus_metrics_enabled? + DummyMetric.new + end + end + def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) - pool.with do |connection| + if prometheus_metrics_enabled? + metrics.map do |metric| + known = [:series, :tags,:values, :timestamp] + value = metric&.[](:values)&.[](:value) + handled= [:rails_gc_statistics] + if handled.include? metric[:series].to_sym + next + end + + if metric.keys.any? {|k| !known.include?(k)} || value.nil? + print metric + print "\n" + + {:series=>"rails_gc_statistics", :tags=>{}, :values=>{:count=>0, :heap_allocated_pages=>4245, :heap_sorted_length=>4426, :heap_allocatable_pages=>0, :heap_available_slots=>1730264, :heap_live_slots=>1729935, :heap_free_slots=>329, :heap_final_slots=>0, :heap_marked_slots=>1184216, :heap_swept_slots=>361843, :heap_eden_pages=>4245, :heap_tomb_pages=>0, :total_allocated_pages=>4245, :total_freed_pages=>0, :total_allocated_objects=>15670757, :total_freed_objects=>13940822, :malloc_increase_bytes=>4842256, :malloc_increase_bytes_limit=>29129457, :minor_gc_count=>0, :major_gc_count=>0, :remembered_wb_unprotected_objects=>39905, :remembered_wb_unprotected_objects_limit=>74474, :old_objects=>1078731, :old_objects_limit=>1975860, :oldmalloc_increase_bytes=>4842640, :oldmalloc_increase_bytes_limit=>31509677, :total_time=>0.0}, :timestamp=>1494356175592659968} + + next + end + metric_value = gauge(metric[:series].to_sym, metric[:series]) + metric_value.set(metric[:tags], value) + end + end + + pool&.with do |connection| prepared.each_slice(settings[:packet_size]) do |slice| begin connection.write_points(slice) @@ -148,7 +207,7 @@ module Gitlab # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. - if enabled? + if influx_metrics_enabled? @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do host = settings[:host] port = settings[:port] diff --git a/lib/gitlab/metrics/dummy_metric.rb b/lib/gitlab/metrics/dummy_metric.rb new file mode 100644 index 00000000000..d27bb83854a --- /dev/null +++ b/lib/gitlab/metrics/dummy_metric.rb @@ -0,0 +1,29 @@ +module Gitlab + module Metrics + # Mocks ::Prometheus::Client::Metric and all derived metrics + class DummyMetric + def get(*args) + raise NotImplementedError + end + + def values(*args) + raise NotImplementedError + end + + # counter + def increment(*args) + # noop + end + + # gauge + def set(*args) + # noop + end + + # histogram / summary + def observe(*args) + # noop + end + end + end +end diff --git a/lib/gitlab/metrics/prometheus_sampler.rb b/lib/gitlab/metrics/prometheus_sampler.rb new file mode 100644 index 00000000000..5f90d4f0b66 --- /dev/null +++ b/lib/gitlab/metrics/prometheus_sampler.rb @@ -0,0 +1,51 @@ +module Gitlab + module Metrics + # Class that sends certain metrics to InfluxDB at a specific interval. + # + # This class is used to gather statistics that can't be directly associated + # with a transaction such as system memory usage, garbage collection + # statistics, etc. + class PrometheusSamples + # interval - The sampling interval in seconds. + def initialize(interval = Metrics.settings[:sample_interval]) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + end + + def start + Thread.new do + Thread.current.abort_on_exception = true + + loop do + sleep(sleep_interval) + + sample + end + end + end + + def sidekiq? + Sidekiq.server? + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end + end + end +end |