1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
|
# frozen_string_literal: true
module Gitlab
module Database
include Gitlab::Metrics::Methods
# The max value of INTEGER type is the same between MySQL and PostgreSQL:
# https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
# http://dev.mysql.com/doc/refman/5.7/en/integer-types.html
MAX_INT_VALUE = 2147483647
# The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz:
# https://www.postgresql.org/docs/9.1/static/datatype-datetime.html
# https://dev.mysql.com/doc/refman/5.7/en/datetime.html
MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze
# Minimum schema version from which migrations are supported
# Migrations before this version may have been removed
MIN_SCHEMA_VERSION = 20190506135400
MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
define_histogram :gitlab_database_transaction_seconds do
docstring "Time spent in database transactions, in seconds"
end
def self.config
ActiveRecord::Base.configurations[Rails.env]
end
def self.username
config['username'] || ENV['USER']
end
def self.database_name
config['database']
end
def self.adapter_name
config['adapter']
end
def self.human_adapter_name
postgresql? ? 'PostgreSQL' : 'MySQL'
end
def self.mysql?
adapter_name.casecmp('mysql2').zero?
end
def self.postgresql?
adapter_name.casecmp('postgresql').zero?
end
def self.read_only?
false
end
def self.read_write?
!self.read_only?
end
# Check whether the underlying database is in read-only mode
def self.db_read_only?
if postgresql?
pg_is_in_recovery =
ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()')
.first.fetch('pg_is_in_recovery')
Gitlab::Utils.to_boolean(pg_is_in_recovery)
else
false
end
end
def self.db_read_write?
!self.db_read_only?
end
def self.version
@version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
end
def self.postgresql_9_or_less?
postgresql? && version.to_f < 10
end
def self.join_lateral_supported?
postgresql? && version.to_f >= 9.3
end
def self.replication_slots_supported?
postgresql? && version.to_f >= 9.4
end
def self.postgresql_minimum_supported_version?
postgresql? && version.to_f >= 9.6
end
# map some of the function names that changed between PostgreSQL 9 and 10
# https://wiki.postgresql.org/wiki/New_in_postgres_10
def self.pg_wal_lsn_diff
Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff'
end
def self.pg_current_wal_insert_lsn
Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn'
end
def self.pg_last_wal_receive_lsn
Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn'
end
def self.pg_last_wal_replay_lsn
Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn'
end
def self.pg_last_xact_replay_timestamp
'pg_last_xact_replay_timestamp'
end
def self.nulls_last_order(field, direction = 'ASC')
order = "#{field} #{direction}"
if postgresql?
order = "#{order} NULLS LAST"
else
# `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
# columns. In the (default) ascending order, `0` comes first.
order = "#{field} IS NULL, #{order}" if direction == 'ASC'
end
order
end
def self.nulls_first_order(field, direction = 'ASC')
order = "#{field} #{direction}"
if postgresql?
order = "#{order} NULLS FIRST"
else
# `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
# columns. In the (default) ascending order, `0` comes first.
order = "#{field} IS NULL, #{order}" if direction == 'DESC'
end
order
end
def self.random
postgresql? ? "RANDOM()" : "RAND()"
end
def self.true_value
if postgresql?
"'t'"
else
1
end
end
def self.false_value
if postgresql?
"'f'"
else
0
end
end
def self.with_connection_pool(pool_size)
pool = create_connection_pool(pool_size)
begin
yield(pool)
ensure
pool.disconnect!
end
end
# Bulk inserts a number of rows into a table, optionally returning their
# IDs.
#
# table - The name of the table to insert the rows into.
# rows - An Array of Hash instances, each mapping the columns to their
# values.
# return_ids - When set to true the return value will be an Array of IDs of
# the inserted rows, this only works on PostgreSQL.
# disable_quote - A key or an Array of keys to exclude from quoting (You
# become responsible for protection from SQL injection for
# these keys!)
def self.bulk_insert(table, rows, return_ids: false, disable_quote: [])
return if rows.empty?
keys = rows.first.keys
columns = keys.map { |key| connection.quote_column_name(key) }
return_ids = false if mysql?
disable_quote = Array(disable_quote).to_set
tuples = rows.map do |row|
keys.map do |k|
disable_quote.include?(k) ? row[k] : connection.quote(row[k])
end
end
sql = <<-EOF
INSERT INTO #{table} (#{columns.join(', ')})
VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
EOF
if return_ids
sql = "#{sql}RETURNING id"
end
result = connection.execute(sql)
if return_ids
result.values.map { |tuple| tuple[0].to_i }
else
[]
end
end
def self.sanitize_timestamp(timestamp)
MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup
end
# pool_size - The size of the DB pool.
# host - An optional host name to use instead of the default one.
def self.create_connection_pool(pool_size, host = nil)
# See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb
env = Rails.env
original_config = ActiveRecord::Base.configurations
env_config = original_config[env].merge('pool' => pool_size)
env_config['host'] = host if host
config = original_config.merge(env => env_config)
spec =
ActiveRecord::
ConnectionAdapters::
ConnectionSpecification::Resolver.new(config).spec(env.to_sym)
ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec)
end
def self.connection
ActiveRecord::Base.connection
end
private_class_method :connection
def self.cached_column_exists?(table_name, column_name)
connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s)
end
def self.cached_table_exists?(table_name)
connection.schema_cache.data_source_exists?(table_name)
end
def self.database_version
row = connection.execute("SELECT VERSION()").first
if postgresql?
row['version']
else
row.first
end
end
private_class_method :database_version
def self.add_post_migrate_path_to_rails(force: false)
return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
Rails.application.config.paths['db'].each do |db_path|
path = Rails.root.join(db_path, 'post_migrate').to_s
unless Rails.application.config.paths['db/migrate'].include? path
Rails.application.config.paths['db/migrate'] << path
# Rails memoizes migrations at certain points where it won't read the above
# path just yet. As such we must also update the following list of paths.
ActiveRecord::Migrator.migrations_paths << path
end
end
end
# inside_transaction? will return true if the caller is running within a transaction. Handles special cases
# when running inside a test environment, in which the entire test is running with a DatabaseCleaner transaction
def self.inside_transaction?
ActiveRecord::Base.connection.open_transactions > open_transactions_baseline
end
def self.open_transactions_baseline
if ::Rails.env.test?
return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) }
end
0
end
private_class_method :open_transactions_baseline
# Monkeypatch rails with upgraded database observability
def self.install_monkey_patches
ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics)
end
# observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to
# record transaction durations.
def self.observe_transaction_duration(duration_seconds)
labels = Gitlab::Metrics::Transaction.current&.labels || {}
gitlab_database_transaction_seconds.observe(labels, duration_seconds)
rescue Prometheus::Client::LabelSetValidator::LabelSetError => err
# Ensure that errors in recording these metrics don't affect the operation of the application
Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger
end
# MonkeyPatch for ActiveRecord::Base for adding observability
module ActiveRecordBaseTransactionMetrics
# A monkeypatch over ActiveRecord::Base.transaction.
# It provides observability into transactional methods.
def transaction(options = {}, &block)
start_time = Gitlab::Metrics::System.monotonic_time
super(options, &block)
ensure
Gitlab::Database.observe_transaction_duration(Gitlab::Metrics::System.monotonic_time - start_time)
end
end
end
end
|