From cfea48dffd04918e4d457ed92ff987b8246ef4ec Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Mon, 29 Jul 2019 11:53:12 +0000 Subject: Adds direct monitoring for sidekiq metrics This adds diirect monitoring for sidekiq metrics. This is done via sidekiq middleware and a sampler to pull from sidekiqs api. --- lib/gitlab/sidekiq_middleware/metrics.rb | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 lib/gitlab/sidekiq_middleware/metrics.rb (limited to 'lib') diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb new file mode 100644 index 00000000000..b06ffa9c121 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class Metrics + def initialize + @metrics = init_metrics + end + + def call(_worker, job, queue) + labels = create_labels(queue) + @metrics[:sidekiq_running_jobs].increment(labels, 1) + + if job['retry_count'].present? + @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) + end + + realtime = Benchmark.realtime do + yield + end + + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime) + rescue Exception # rubocop: disable Lint/RescueException + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) + raise + ensure + @metrics[:sidekiq_running_jobs].increment(labels, -1) + end + + private + + def init_metrics + { + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job'), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum) + } + end + + def create_labels(queue) + { + queue: queue + } + end + end + end +end -- cgit v1.2.1