diff options
Diffstat (limited to 'lib/gitlab/metrics')
-rw-r--r-- | lib/gitlab/metrics/exporter/base_exporter.rb | 45 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/gc_request_middleware.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/health_checks_middleware.rb | 35 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/metrics_middleware.rb | 41 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/sidekiq_exporter.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/web_exporter.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/action_cable_sampler.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/base_sampler.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 4 |
9 files changed, 138 insertions, 43 deletions
diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 47c862c0232..190d3d3fd2f 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -11,44 +11,41 @@ module Gitlab attr_accessor :readiness_checks - def initialize(settings, **options) + def initialize(settings, log_enabled:, log_file:, gc_requests: false, **options) super(**options) @settings = settings + @gc_requests = gc_requests + + # log_enabled does not exist for all exporters + log_sink = log_enabled ? File.join(Rails.root, 'log', log_file) : File::NULL + @logger = WEBrick::Log.new(log_sink) + @logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" end def enabled? settings.enabled end - def log_filename - raise NotImplementedError - end - private - attr_reader :settings + attr_reader :settings, :logger def start_working - logger = WEBrick::Log.new(log_filename) - logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" - access_log = [ [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] ] @server = ::WEBrick::HTTPServer.new( Port: settings.port, BindAddress: settings.address, - Logger: logger, AccessLog: access_log) - server.mount_proc '/readiness' do |req, res| - render_probe(readiness_probe, req, res) - end - server.mount_proc '/liveness' do |req, res| - render_probe(liveness_probe, req, res) - end + Logger: logger, AccessLog: access_log + ) server.mount '/', Rack::Handler::WEBrick, rack_app true + rescue StandardError => e + logger.error(e) + false end def run_thread @@ -72,8 +69,16 @@ module Gitlab end def rack_app + readiness = readiness_probe + liveness = liveness_probe + pid = thread_name + gc_requests = @gc_requests + Rack::Builder.app do use Rack::Deflater + use Gitlab::Metrics::Exporter::MetricsMiddleware, pid + use Gitlab::Metrics::Exporter::HealthChecksMiddleware, readiness, liveness + use Gitlab::Metrics::Exporter::GcRequestMiddleware if gc_requests use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present? run -> (env) { [404, {}, ['']] } end @@ -86,14 +91,6 @@ module Gitlab def liveness_probe ::Gitlab::HealthChecks::Probes::Collection.new end - - def render_probe(probe, req, res) - result = probe.execute - - res.status = result.http_status - res.content_type = 'application/json; charset=utf-8' - res.body = result.json.to_json - end end end end diff --git a/lib/gitlab/metrics/exporter/gc_request_middleware.rb b/lib/gitlab/metrics/exporter/gc_request_middleware.rb new file mode 100644 index 00000000000..3806b0e2bd1 --- /dev/null +++ b/lib/gitlab/metrics/exporter/gc_request_middleware.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class GcRequestMiddleware + def initialize(app) + @app = app + end + + def call(env) + @app.call(env).tap do + GC.start + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/health_checks_middleware.rb b/lib/gitlab/metrics/exporter/health_checks_middleware.rb new file mode 100644 index 00000000000..c43b8004b72 --- /dev/null +++ b/lib/gitlab/metrics/exporter/health_checks_middleware.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class HealthChecksMiddleware + def initialize(app, readiness_probe, liveness_probe) + @app = app + @readiness_probe = readiness_probe + @liveness_probe = liveness_probe + end + + def call(env) + case env['PATH_INFO'] + when '/readiness' then render_probe(@readiness_probe) + when '/liveness' then render_probe(@liveness_probe) + else @app.call(env) + end + end + + private + + def render_probe(probe) + result = probe.execute + + [ + result.http_status, + { 'Content-Type' => 'application/json; charset=utf-8' }, + [result.json.to_json] + ] + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb new file mode 100644 index 00000000000..e17f1c13cf0 --- /dev/null +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class MetricsMiddleware + def initialize(app, pid) + @app = app + default_labels = { + pid: pid + } + @requests_total = Gitlab::Metrics.counter( + :exporter_http_requests_total, 'Total number of HTTP requests', default_labels + ) + @request_durations = Gitlab::Metrics.histogram( + :exporter_http_request_duration_seconds, + 'HTTP request duration histogram (seconds)', + default_labels, + [0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + ) + end + + def call(env) + start = Gitlab::Metrics::System.monotonic_time + @app.call(env).tap do |response| + duration = Gitlab::Metrics::System.monotonic_time - start + + labels = { + method: env['REQUEST_METHOD'].downcase, + path: env['PATH_INFO'].to_s, + code: response.first.to_s + } + + @requests_total.increment(labels) + @request_durations.observe(labels, duration) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index eea71fda6a0..afecf6546f8 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -4,12 +4,11 @@ module Gitlab module Metrics module Exporter class SidekiqExporter < BaseExporter - def log_filename - if settings['log_enabled'] - File.join(Rails.root, 'log', 'sidekiq_exporter.log') - else - File::NULL - end + def initialize(settings, **options) + super(settings, + log_enabled: settings['log_enabled'], + log_file: 'sidekiq_exporter.log', + **options) end end end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index d41484aaaa7..c05ad8ccf42 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -26,8 +26,8 @@ module Gitlab attr_reader :running # This exporter is always run on master process - def initialize - super(Settings.monitoring.web_exporter) + def initialize(**options) + super(Settings.monitoring.web_exporter, log_enabled: true, log_file: 'web_exporter.log', **options) # DEPRECATED: # these `readiness_checks` are deprecated @@ -39,10 +39,6 @@ module Gitlab ] end - def log_filename - File.join(Rails.root, 'log', 'web_exporter.log') - end - def mark_as_not_running! @running = false end diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb index adce3030d0d..1f50371cae9 100644 --- a/lib/gitlab/metrics/samplers/action_cable_sampler.rb +++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb @@ -6,8 +6,8 @@ module Gitlab class ActionCableSampler < BaseSampler DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 - def initialize(interval = nil, action_cable: ::ActionCable.server) - super(interval) + def initialize(action_cable: ::ActionCable.server, **options) + super(**options) @action_cable = action_cable end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 52d80c3c27e..b2a9de21145 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -9,7 +9,10 @@ module Gitlab attr_reader :interval # interval - The sampling interval in seconds. - def initialize(interval = nil) + # warmup - When true, takes a single sample eagerly before entering the sampling loop. + # This can be useful to ensure that all metrics files exist after `start` returns, + # since prometheus-client-mmap creates them lazily upon first access. + def initialize(interval: nil, logger: Logger.new($stdout), warmup: false, **options) interval ||= ENV[interval_env_key]&.to_i interval ||= self.class::DEFAULT_SAMPLING_INTERVAL_SECONDS interval_half = interval.to_f / 2 @@ -17,13 +20,16 @@ module Gitlab @interval = interval @interval_steps = (-interval_half..interval_half).step(0.1).to_a - super() + @logger = logger + @warmup = warmup + + super(**options) end def safe_sample sample rescue StandardError => e - ::Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") + @logger.warn("#{self.class}: #{e}, stopping") stop end @@ -63,6 +69,8 @@ module Gitlab def start_working @running = true + safe_sample if @warmup + true end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index b1c5e9800da..d71ee671b8d 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -7,12 +7,12 @@ module Gitlab DEFAULT_SAMPLING_INTERVAL_SECONDS = 60 GC_REPORT_BUCKETS = [0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 1].freeze - def initialize(*) + def initialize(...) GC::Profiler.clear metrics[:process_start_time_seconds].set(labels, Time.now.to_i) - super + super(...) end def metrics |