diff options
| -rw-r--r-- | changelogs/unreleased/pawel-add-sidekiq-metrics-endpoint-32145.yml | 4 | ||||
| -rw-r--r-- | config/gitlab.yml.example | 6 | ||||
| -rw-r--r-- | config/initializers/1_settings.rb | 4 | ||||
| -rw-r--r-- | config/initializers/7_prometheus_metrics.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/daemon.rb | 62 | ||||
| -rw-r--r-- | lib/gitlab/metrics/base_sampler.rb | 75 | ||||
| -rw-r--r-- | lib/gitlab/metrics/sidekiq_metrics_exporter.rb | 39 | ||||
| -rw-r--r-- | spec/lib/gitlab/daemon_spec.rb | 103 | ||||
| -rw-r--r-- | spec/lib/gitlab/metrics/influx_sampler_spec.rb | 2 | ||||
| -rw-r--r-- | spec/lib/gitlab/metrics/sidekiq_metrics_exporter_spec.rb | 101 |
10 files changed, 348 insertions, 54 deletions
diff --git a/changelogs/unreleased/pawel-add-sidekiq-metrics-endpoint-32145.yml b/changelogs/unreleased/pawel-add-sidekiq-metrics-endpoint-32145.yml new file mode 100644 index 00000000000..71eabdc16d2 --- /dev/null +++ b/changelogs/unreleased/pawel-add-sidekiq-metrics-endpoint-32145.yml @@ -0,0 +1,4 @@ +--- +title: Add Prometheus metrics exporter to Sidekiq +merge_request: 13082 +author: diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 45ab4e1a851..e73db08fcac 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -590,6 +590,12 @@ production: &base ip_whitelist: - 127.0.0.0/8 + # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics + sidekiq_exporter: + # enabled: true + # address: localhost + # port: 3807 + # # 5. Extra customization # ========================== diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 7a43bf939ea..e24cf33adb5 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -524,6 +524,10 @@ Settings.webpack.dev_server['port'] ||= 3808 Settings['monitoring'] ||= Settingslogic.new({}) Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8'] Settings.monitoring['unicorn_sampler_interval'] ||= 10 +Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({}) +Settings.monitoring.sidekiq_exporter['enabled'] ||= false +Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost' +Settings.monitoring.sidekiq_exporter['port'] ||= 3807 # # Testing settings diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index a2f8421f5d7..54c797e0714 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -10,3 +10,9 @@ Prometheus::Client.configure do |config| config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir') end end + +Sidekiq.configure_server do |config| + config.on(:startup) do + Gitlab::Metrics::SidekiqMetricsExporter.instance.start + end +end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb new file mode 100644 index 00000000000..dfd17e35707 --- /dev/null +++ b/lib/gitlab/daemon.rb @@ -0,0 +1,62 @@ +module Gitlab + class Daemon + def self.initialize_instance(*args) + raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) + Kernel.at_exit(&@instance.method(:stop)) + @instance + end + + def self.instance + @instance ||= initialize_instance + end + + attr_reader :thread + + def thread? + !thread.nil? + end + + def initialize + @mutex = Mutex.new + end + + def enabled? + true + end + + def start + return unless enabled? + + @mutex.synchronize do + return thread if thread? + + @thread = Thread.new { start_working } + end + end + + def stop + @mutex.synchronize do + return unless thread? + + stop_working + + if thread + thread.wakeup if thread.alive? + thread.join + @thread = nil + end + end + end + + private + + def start_working + raise NotImplementedError + end + + def stop_working + # no-ops + end + end +end diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb index 219accfc029..716d20bb91a 100644 --- a/lib/gitlab/metrics/base_sampler.rb +++ b/lib/gitlab/metrics/base_sampler.rb @@ -1,20 +1,7 @@ require 'logger' module Gitlab module Metrics - class BaseSampler - def self.initialize_instance(*args) - raise "#{name} singleton instance already initialized" if @instance - @instance = new(*args) - at_exit(&@instance.method(:stop)) - @instance - end - - def self.instance - @instance - end - - attr_reader :running - + class BaseSampler < Daemon # interval - The sampling interval in seconds. def initialize(interval) interval_half = interval.to_f / 2 @@ -22,44 +9,7 @@ module Gitlab @interval = interval @interval_steps = (-interval_half..interval_half).step(0.1).to_a - @mutex = Mutex.new - end - - def enabled? - true - end - - def start - return unless enabled? - - @mutex.synchronize do - return if running - @running = true - - @thread = Thread.new do - sleep(sleep_interval) - - while running - safe_sample - - sleep(sleep_interval) - end - end - end - end - - def stop - @mutex.synchronize do - return unless running - - @running = false - - if @thread - @thread.wakeup if @thread.alive? - @thread.join - @thread = nil - end - end + super() end def safe_sample @@ -81,7 +31,7 @@ module Gitlab # potentially missing anything that happens in between samples). # 2. Don't sample data at the same interval two times in a row. def sleep_interval - while step = @interval_steps.sample + while (step = @interval_steps.sample) if step != @last_step @last_step = step @@ -89,6 +39,25 @@ module Gitlab end end end + + private + + attr_reader :running + + def start_working + @running = true + sleep(sleep_interval) + + while running + safe_sample + + sleep(sleep_interval) + end + end + + def stop_working + @running = false + end end end end diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb new file mode 100644 index 00000000000..5980a4ded2b --- /dev/null +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -0,0 +1,39 @@ +require 'webrick' +require 'prometheus/client/rack/exporter' + +module Gitlab + module Metrics + class SidekiqMetricsExporter < Daemon + def enabled? + Gitlab::Metrics.metrics_folder_present? && settings.enabled + end + + def settings + Settings.monitoring.sidekiq_exporter + end + + private + + attr_reader :server + + def start_working + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + server.mount "/", Rack::Handler::WEBrick, rack_app + server.start + end + + def stop_working + server.shutdown + @server = nil + end + + def rack_app + Rack::Builder.app do + use Rack::Deflater + use ::Prometheus::Client::Rack::Exporter + run -> (env) { [404, {}, ['']] } + end + end + end + end +end diff --git a/spec/lib/gitlab/daemon_spec.rb b/spec/lib/gitlab/daemon_spec.rb new file mode 100644 index 00000000000..c519984a267 --- /dev/null +++ b/spec/lib/gitlab/daemon_spec.rb @@ -0,0 +1,103 @@ +require 'spec_helper' + +describe Gitlab::Daemon do + subject { described_class.new } + + before do + allow(subject).to receive(:start_working) + allow(subject).to receive(:stop_working) + end + + describe '.instance' do + before do + allow(Kernel).to receive(:at_exit) + end + + after(:each) do + described_class.instance_variable_set(:@instance, nil) + end + + it 'provides instance of Daemon' do + expect(described_class.instance).to be_instance_of(described_class) + end + + it 'subsequent invocations provide the same instance' do + expect(described_class.instance).to eq(described_class.instance) + end + + it 'creates at_exit hook when instance is created' do + expect(described_class.instance).not_to be_nil + + expect(Kernel).to have_received(:at_exit) + end + end + + describe 'when Daemon is enabled' do + before do + allow(subject).to receive(:enabled?).and_return(true) + end + + describe 'when Daemon is stopped' do + describe '#start' do + it 'starts the Daemon' do + expect { subject.start.join }.to change { subject.thread? }.from(false).to(true) + + expect(subject).to have_received(:start_working) + end + end + + describe '#stop' do + it "doesn't shutdown stopped Daemon" do + expect { subject.stop }.not_to change { subject.thread? } + + expect(subject).not_to have_received(:start_working) + end + end + end + + describe 'when Daemon is running' do + before do + subject.start.join + end + + describe '#start' do + it "doesn't start running Daemon" do + expect { subject.start.join }.not_to change { subject.thread? } + + expect(subject).to have_received(:start_working).once + end + end + + describe '#stop' do + it 'shutdowns Daemon' do + expect { subject.stop }.to change { subject.thread? }.from(true).to(false) + + expect(subject).to have_received(:stop_working) + end + end + end + end + + describe 'when Daemon is disabled' do + before do + allow(subject).to receive(:enabled?).and_return(false) + end + + describe '#start' do + it "doesn't start working" do + expect(subject.start).to be_nil + expect { subject.start }.not_to change { subject.thread? } + + expect(subject).not_to have_received(:start_working) + end + end + + describe '#stop' do + it "doesn't stop working" do + expect { subject.stop }.not_to change { subject.thread? } + + expect(subject).not_to have_received(:stop_working) + end + end + end +end diff --git a/spec/lib/gitlab/metrics/influx_sampler_spec.rb b/spec/lib/gitlab/metrics/influx_sampler_spec.rb index 0bc68d64276..999a9536d82 100644 --- a/spec/lib/gitlab/metrics/influx_sampler_spec.rb +++ b/spec/lib/gitlab/metrics/influx_sampler_spec.rb @@ -11,7 +11,7 @@ describe Gitlab::Metrics::InfluxSampler do it 'runs once and gathers a sample at a given interval' do expect(sampler).to receive(:sleep).with(a_kind_of(Numeric)).twice expect(sampler).to receive(:sample).once - expect(sampler).to receive(:running).and_return(false, true, false) + expect(sampler).to receive(:running).and_return(true, false) sampler.start.join end diff --git a/spec/lib/gitlab/metrics/sidekiq_metrics_exporter_spec.rb b/spec/lib/gitlab/metrics/sidekiq_metrics_exporter_spec.rb new file mode 100644 index 00000000000..6721e02fb85 --- /dev/null +++ b/spec/lib/gitlab/metrics/sidekiq_metrics_exporter_spec.rb @@ -0,0 +1,101 @@ +require 'spec_helper' + +describe Gitlab::Metrics::SidekiqMetricsExporter do + let(:exporter) { described_class.new } + let(:server) { double('server') } + + before do + allow(::WEBrick::HTTPServer).to receive(:new).and_return(server) + allow(server).to receive(:mount) + allow(server).to receive(:start) + allow(server).to receive(:shutdown) + end + + describe 'when exporter is enabled' do + before do + allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(true) + end + + describe 'when exporter is stopped' do + describe '#start' do + it 'starts the exporter' do + expect { exporter.start.join }.to change { exporter.thread? }.from(false).to(true) + + expect(server).to have_received(:start) + end + + describe 'with custom settings' do + let(:port) { 99999 } + let(:address) { 'sidekiq_exporter_address' } + + before do + allow(Settings.monitoring.sidekiq_exporter).to receive(:port).and_return(port) + allow(Settings.monitoring.sidekiq_exporter).to receive(:address).and_return(address) + end + + it 'starts server with port and address from settings' do + exporter.start.join + + expect(::WEBrick::HTTPServer).to have_received(:new).with( + Port: port, + BindAddress: address + ) + end + end + end + + describe '#stop' do + it "doesn't shutdown stopped server" do + expect { exporter.stop }.not_to change { exporter.thread? } + + expect(server).not_to have_received(:shutdown) + end + end + end + + describe 'when exporter is running' do + before do + exporter.start.join + end + + describe '#start' do + it "doesn't start running server" do + expect { exporter.start.join }.not_to change { exporter.thread? } + + expect(server).to have_received(:start).once + end + end + + describe '#stop' do + it 'shutdowns server' do + expect { exporter.stop }.to change { exporter.thread? }.from(true).to(false) + + expect(server).to have_received(:shutdown) + end + end + end + end + + describe 'when exporter is disabled' do + before do + allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(false) + end + + describe '#start' do + it "doesn't start" do + expect(exporter.start).to be_nil + expect { exporter.start }.not_to change { exporter.thread? } + + expect(server).not_to have_received(:start) + end + end + + describe '#stop' do + it "doesn't shutdown" do + expect { exporter.stop }.not_to change { exporter.thread? } + + expect(server).not_to have_received(:shutdown) + end + end + end +end |