1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
# frozen_string_literal: true
require_relative '../config/boot'
class MetricsServer # rubocop:disable Gitlab/NamespacedClass
# The singleton instance used to supervise the Puma metrics server.
PumaProcessSupervisor = Class.new(Gitlab::ProcessSupervisor)
class << self
def start_for_puma
metrics_dir = ::Prometheus::Client.configuration.multiprocess_files_dir
start_server = proc do
MetricsServer.spawn('puma', metrics_dir: metrics_dir).tap do |pid|
Gitlab::AppLogger.info("Starting Puma metrics server with pid #{pid}")
end
end
supervisor = PumaProcessSupervisor.instance
supervisor.supervise(start_server.call) do
next unless supervisor.alive
Gitlab::AppLogger.info('Puma metrics server terminated, restarting...')
start_server.call
end
end
def spawn(target, metrics_dir:, gitlab_config: nil, wipe_metrics_dir: false)
ensure_valid_target!(target)
cmd = "#{Rails.root}/bin/metrics-server"
env = {
'METRICS_SERVER_TARGET' => target,
'WIPE_METRICS_DIR' => wipe_metrics_dir ? '1' : '0'
}
env['GITLAB_CONFIG'] = gitlab_config if gitlab_config
Process.spawn(env, cmd, err: $stderr, out: $stdout, pgroup: true).tap do |pid|
Process.detach(pid)
end
end
def fork(target, metrics_dir:, wipe_metrics_dir: false, reset_signals: [])
ensure_valid_target!(target)
pid = Process.fork
if pid.nil? # nil means we're inside the fork
# Remove any custom signal handlers the parent process had registered, since we do
# not want to inherit them, and Ruby forks with a `clone` that has the `CLONE_SIGHAND`
# flag set.
Gitlab::ProcessManagement.modify_signals(reset_signals, 'DEFAULT')
server = MetricsServer.new(target, metrics_dir, wipe_metrics_dir)
# This rewrites /proc/cmdline, since otherwise tools like `top` will show the
# parent process `cmdline` which is really confusing.
$0 = server.name
server.start
else
Process.detach(pid)
end
pid
end
private
def ensure_valid_target!(target)
raise "Target must be one of [puma,sidekiq]" unless %w(puma sidekiq).include?(target)
end
end
def initialize(target, metrics_dir, wipe_metrics_dir)
@target = target
@metrics_dir = metrics_dir
@wipe_metrics_dir = wipe_metrics_dir
end
def start
::Prometheus::Client.configure do |config|
config.multiprocess_files_dir = @metrics_dir
config.pid_provider = proc { name }
end
FileUtils.mkdir_p(@metrics_dir, mode: 0700)
::Prometheus::CleanupMultiprocDirService.new(@metrics_dir).execute if @wipe_metrics_dir
# We need to `warmup: true` since otherwise the sampler and exporter threads enter
# a race where not all Prometheus db files will be visible to the exporter, resulting
# in missing metrics.
# Warming up ensures that these files exist prior to the exporter starting up.
Gitlab::Metrics::Samplers::RubySampler.initialize_instance(prefix: name, warmup: true).start
default_opts = { gc_requests: true, synchronous: true }
exporter =
case @target
when 'puma'
Gitlab::Metrics::Exporter::WebExporter.instance(**default_opts)
when 'sidekiq'
settings = Settings.new(Settings.monitoring[name])
Gitlab::Metrics::Exporter::SidekiqExporter.instance(settings, **default_opts)
end
exporter.start
end
def name
case @target
when 'puma' then 'web_exporter'
when 'sidekiq' then 'sidekiq_exporter'
end
end
end
|
