diff options
author | Stan Hu <stanhu@gmail.com> | 2019-08-12 16:20:31 +0000 |
---|---|---|
committer | Stan Hu <stanhu@gmail.com> | 2019-08-12 16:20:31 +0000 |
commit | 5971cd15086f65fa7202b05b26fc2f980b9f2767 (patch) | |
tree | 65d28e7772ba68386d2ecf4d3a3b2c7ef79f8a58 | |
parent | 7a22b4bae55ba3afcc6fc961e6a0aeb47c7bd937 (diff) | |
parent | 35fb27db277be62dde1f5c58709fd6af59c144a9 (diff) | |
download | gitlab-ce-5971cd15086f65fa7202b05b26fc2f980b9f2767.tar.gz |
Merge branch '65278-fix-puma-master-counter-wipe' into 'master'
Fix active metric files being wiped after the app starts
See merge request gitlab-org/gitlab-ce!31668
-rw-r--r-- | changelogs/unreleased/65278-fix-puma-master-counter-wipe.yml | 5 | ||||
-rw-r--r-- | config.ru | 18 | ||||
-rw-r--r-- | config/initializers/7_prometheus_metrics.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/cluster/puma_worker_killer_observer.rb | 4 | ||||
-rw-r--r-- | spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb | 4 |
5 files changed, 25 insertions, 25 deletions
diff --git a/changelogs/unreleased/65278-fix-puma-master-counter-wipe.yml b/changelogs/unreleased/65278-fix-puma-master-counter-wipe.yml new file mode 100644 index 00000000000..fb9d6fa251d --- /dev/null +++ b/changelogs/unreleased/65278-fix-puma-master-counter-wipe.yml @@ -0,0 +1,5 @@ +--- +title: Fix active metric files being wiped after the app starts +merge_request: 31668 +author: +type: fixed diff --git a/config.ru b/config.ru index 6f6fb85d8fa..f6a7dca0542 100644 --- a/config.ru +++ b/config.ru @@ -17,7 +17,25 @@ end require ::File.expand_path('../config/environment', __FILE__) +# The following is necessary to ensure stale Prometheus metrics don't accumulate over time. +# It needs to be done as early as here to ensure metrics files aren't deleted. +# After we hit our app in `warmup`, first metrics and corresponding files already being created, +# for example in `lib/gitlab/metrics/requests_rack_middleware.rb`. +def cleanup_prometheus_multiproc_dir + if dir = ::Prometheus::Client.configuration.multiprocess_files_dir + old_metrics = Dir[File.join(dir, '*.db')] + + FileUtils.rm_rf(old_metrics) + end +end + +def master_process? + Prometheus::PidProvider.worker_id.in? %w(unicorn_master puma_master) +end + warmup do |app| + cleanup_prometheus_multiproc_dir if master_process? + client = Rack::MockRequest.new(app) client.get('/') end diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 3f2691dde95..70e5dcd042e 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -51,22 +51,3 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled? end end end - -def cleanup_prometheus_multiproc_dir - # The following is necessary to ensure stale Prometheus metrics don't - # accumulate over time. It needs to be done in this hook as opposed to - # inside an init script to ensure metrics files aren't deleted after new - # unicorn workers start after a SIGUSR2 is received. - if dir = ::Prometheus::Client.configuration.multiprocess_files_dir - old_metrics = Dir[File.join(dir, '*.db')] - FileUtils.rm_rf(old_metrics) - end -end - -Gitlab::Cluster::LifecycleEvents.on_master_start do - cleanup_prometheus_multiproc_dir -end - -Gitlab::Cluster::LifecycleEvents.on_master_restart do - cleanup_prometheus_multiproc_dir -end diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb index 3b4ebc3fbae..f53051c32ff 100644 --- a/lib/gitlab/cluster/puma_worker_killer_observer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_observer.rb @@ -15,9 +15,7 @@ module Gitlab private def log_termination(worker) - labels = { worker: "worker_#{worker.index}" } - - @counter.increment(labels) + @counter.increment end end end diff --git a/spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb b/spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb index 180520b27e7..6ed9dda08d7 100644 --- a/spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb +++ b/spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb @@ -17,9 +17,7 @@ describe Gitlab::Cluster::PumaWorkerKillerObserver do it 'increments timeout counter' do worker = double(index: 0) - expect(counter) - .to receive(:increment) - .with({ worker: 'worker_0' }) + expect(counter).to receive(:increment) subject.callback.call(worker) end |