1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
# frozen_string_literal: true
module Gitlab
module Metrics
module Subscribers
# Class for tracking the total query duration of a transaction.
class ActiveRecord < ActiveSupport::Subscriber
extend Gitlab::Utils::StrongMemoize
attach_to :active_record
DB_COUNTERS = %i{count write_count cached_count}.freeze
SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze
SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze
TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze
DB_LOAD_BALANCING_ROLES = %i{replica primary}.freeze
DB_LOAD_BALANCING_COUNTERS = %i{count cached_count wal_count wal_cached_count}.freeze
DB_LOAD_BALANCING_DURATIONS = %i{duration_s}.freeze
SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze
# This event is published from ActiveRecordBaseTransactionMetrics and
# used to record a database transaction duration when calling
# ApplicationRecord.transaction {} block.
def transaction(event)
observe(:gitlab_database_transaction_seconds, event) do
buckets TRANSACTION_DURATION_BUCKET
end
end
def sql(event)
# Mark this thread as requiring a database connection. This is used
# by the Gitlab::Metrics::Samplers::ThreadsSampler to count threads
# using a connection.
Thread.current[:uses_db_connection] = true
payload = event.payload
return if ignored_query?(payload)
db_config_name = db_config_name(event.payload)
increment(:count, db_config_name: db_config_name)
increment(:cached_count, db_config_name: db_config_name) if cached_query?(payload)
increment(:write_count, db_config_name: db_config_name) unless select_sql_command?(payload)
observe(:gitlab_sql_duration_seconds, event) do
buckets SQL_DURATION_BUCKET
end
db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection])
return if db_role.blank?
increment_db_role_counters(db_role, payload)
observe_db_role_duration(db_role, event)
end
def self.db_counter_payload
return {} unless Gitlab::SafeRequestStore.active?
{}.tap do |payload|
db_counter_keys.each do |key|
payload[key] = Gitlab::SafeRequestStore[key].to_i
end
if ::Gitlab::SafeRequestStore.active?
load_balancing_metric_counter_keys.each do |counter|
payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i
end
load_balancing_metric_duration_keys.each do |duration|
payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3)
end
end
end
end
private
def wal_command?(payload)
payload[:sql].match(SQL_WAL_LOCATION_REGEX)
end
def increment_db_role_counters(db_role, payload)
cached = cached_query?(payload)
db_config_name = db_config_name(payload)
increment(:count, db_role: db_role, db_config_name: db_config_name)
increment(:cached_count, db_role: db_role, db_config_name: db_config_name) if cached
if wal_command?(payload)
increment(:wal_count, db_role: db_role, db_config_name: db_config_name)
increment(:wal_cached_count, db_role: db_role, db_config_name: db_config_name) if cached
end
end
def observe_db_role_duration(db_role, event)
observe("gitlab_sql_#{db_role}_duration_seconds".to_sym, event) do
buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET
end
return unless ::Gitlab::SafeRequestStore.active?
duration = event.duration / 1000.0
duration_key = compose_metric_key(:duration_s, db_role)
::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration
# Per database metrics
db_config_name = db_config_name(event.payload)
duration_key = compose_metric_key(:duration_s, nil, db_config_name)
::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration
end
def ignored_query?(payload)
payload[:name] == 'SCHEMA' || payload[:name] == 'TRANSACTION'
end
def cached_query?(payload)
payload.fetch(:cached, payload[:name] == 'CACHE')
end
def select_sql_command?(payload)
payload[:sql].match(SQL_COMMANDS_WITH_COMMENTS_REGEX)
end
def increment(counter, db_config_name:, db_role: nil)
log_key = compose_metric_key(counter, db_role)
prometheus_key = if db_role
:"gitlab_transaction_db_#{db_role}_#{counter}_total"
else
:"gitlab_transaction_db_#{counter}_total"
end
current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name })
Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1
# To avoid confusing log keys we only log the db_config_name metrics
# when we are also logging the db_role. Otherwise it will be hard to
# tell if the log key is referring to a db_role OR a db_config_name.
if db_role.present? && db_config_name.present?
log_key = compose_metric_key(counter, nil, db_config_name)
Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1
end
end
def observe(histogram, event, &block)
db_config_name = db_config_name(event.payload)
current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block)
end
def current_transaction
::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current
end
def db_config_name(payload)
::Gitlab::Database.db_config_name(payload[:connection])
end
def self.db_counter_keys
DB_COUNTERS.map { |c| compose_metric_key(c) }
end
def self.load_balancing_metric_counter_keys
strong_memoize(:load_balancing_metric_counter_keys) do
load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS)
end
end
def self.load_balancing_metric_duration_keys
strong_memoize(:load_balancing_metric_duration_keys) do
load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS)
end
end
def self.load_balancing_metric_keys(metrics)
counters = []
metrics.each do |metric|
DB_LOAD_BALANCING_ROLES.each do |role|
counters << compose_metric_key(metric, role)
end
::Gitlab::Database.database_base_models.keys.each do |config_name|
counters << compose_metric_key(metric, nil, config_name) # main / ci / geo
end
::Gitlab::Database.database_base_models_using_load_balancing.keys.each do |config_name|
counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica / ci_replica
end
end
counters
end
private_class_method :load_balancing_metric_keys
def compose_metric_key(metric, db_role = nil, db_config_name = nil)
self.class.compose_metric_key(metric, db_role, db_config_name)
end
def self.compose_metric_key(metric, db_role = nil, db_config_name = nil)
[:db, db_role, db_config_name, metric].compact.join("_").to_sym
end
end
end
end
end
|
