1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
|
# frozen_string_literal: true
class ContainerRepository < ApplicationRecord
include Gitlab::Utils::StrongMemoize
include Gitlab::SQL::Pattern
include EachBatch
include Sortable
include AfterCommitQueue
WAITING_CLEANUP_STATUSES = %i[cleanup_scheduled cleanup_unfinished].freeze
REQUIRING_CLEANUP_STATUSES = %i[cleanup_unscheduled cleanup_scheduled].freeze
IDLE_MIGRATION_STATES = %w[default pre_import_done import_done import_aborted import_skipped].freeze
ACTIVE_MIGRATION_STATES = %w[pre_importing importing].freeze
ABORTABLE_MIGRATION_STATES = (ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze
MIGRATION_STATES = (IDLE_MIGRATION_STATES + ACTIVE_MIGRATION_STATES).freeze
TooManyImportsError = Class.new(StandardError)
NativeImportError = Class.new(StandardError)
belongs_to :project
validates :name, length: { minimum: 0, allow_nil: false }
validates :name, uniqueness: { scope: :project_id }
validates :migration_state, presence: true, inclusion: { in: MIGRATION_STATES }
validates :migration_aborted_in_state, inclusion: { in: ABORTABLE_MIGRATION_STATES }, allow_nil: true
validates :migration_retries_count, presence: true,
numericality: { greater_than_or_equal_to: 0 },
allow_nil: false
enum status: { delete_scheduled: 0, delete_failed: 1 }
enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 }
enum migration_skipped_reason: { not_in_plan: 0, too_many_retries: 1, too_many_tags: 2, root_namespace_in_deny_list: 3 }
delegate :client, :gitlab_api_client, to: :registry
scope :ordered, -> { order(:name) }
scope :with_api_entity_associations, -> { preload(project: [:route, { namespace: :route }]) }
scope :for_group_and_its_subgroups, ->(group) do
project_scope = Project
.for_group_and_its_subgroups(group)
.with_feature_enabled(:container_registry)
.select(:id)
joins("INNER JOIN (#{project_scope.to_sql}) projects on projects.id=container_repositories.project_id")
end
scope :for_project_id, ->(project_id) { where(project_id: project_id) }
scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) }
scope :waiting_for_cleanup, -> { where(expiration_policy_cleanup_status: WAITING_CLEANUP_STATUSES) }
scope :expiration_policy_started_at_nil_or_before, ->(timestamp) { where('expiration_policy_started_at < ? OR expiration_policy_started_at IS NULL', timestamp) }
scope :with_migration_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_import_started_at, '01-01-1970') < ?", timestamp) }
scope :with_migration_pre_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_started_at, '01-01-1970') < ?", timestamp) }
scope :with_migration_pre_import_done_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_done_at, '01-01-1970') < ?", timestamp) }
scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.where('expiration_policy_started_at < ?', threshold) }
scope :import_in_process, -> { where(migration_state: %w[pre_importing pre_import_done importing]) }
scope :recently_done_migration_step, -> do
where(migration_state: %w[import_done pre_import_done import_aborted])
.order(Arel.sql('GREATEST(migration_pre_import_done_at, migration_import_done_at, migration_aborted_at) DESC'))
end
scope :ready_for_import, -> do
# There is no yaml file for the container_registry_phase_2_deny_list
# feature flag since it is only accessed in this query.
# https://gitlab.com/gitlab-org/gitlab/-/issues/350543 tracks the rollout and
# removal of this feature flag.
joins(:project).where(
migration_state: [:default],
created_at: ...ContainerRegistry::Migration.created_before
).with_target_import_tier
.where(
"NOT EXISTS (
SELECT 1
FROM feature_gates
WHERE feature_gates.feature_key = 'container_registry_phase_2_deny_list'
AND feature_gates.key = 'actors'
AND feature_gates.value = concat('Group:', projects.namespace_id)
)"
)
end
state_machine :migration_state, initial: :default, use_transactions: false do
state :pre_importing do
validates :migration_pre_import_started_at, presence: true
validates :migration_pre_import_done_at, presence: false
end
state :pre_import_done do
validates :migration_pre_import_done_at, presence: true
end
state :importing do
validates :migration_import_started_at, presence: true
validates :migration_import_done_at, presence: false
end
state :import_done
state :import_skipped do
validates :migration_skipped_reason,
:migration_skipped_at,
presence: true
end
state :import_aborted do
validates :migration_aborted_at, presence: true
validates :migration_retries_count, presence: true, numericality: { greater_than_or_equal_to: 1 }
end
event :start_pre_import do
transition default: :pre_importing
end
event :finish_pre_import do
transition %i[pre_importing import_aborted] => :pre_import_done
end
event :start_import do
transition pre_import_done: :importing
end
event :finish_import do
transition %i[importing import_aborted] => :import_done
end
event :already_migrated do
transition default: :import_done
end
event :abort_import do
transition ABORTABLE_MIGRATION_STATES.map(&:to_sym) => :import_aborted
end
event :skip_import do
transition ABORTABLE_MIGRATION_STATES.map(&:to_sym) => :import_skipped
end
event :retry_pre_import do
transition import_aborted: :pre_importing
end
event :retry_import do
transition import_aborted: :importing
end
before_transition any => :pre_importing do |container_repository|
container_repository.migration_pre_import_started_at = Time.zone.now
container_repository.migration_pre_import_done_at = nil
end
after_transition any => :pre_importing do |container_repository|
container_repository.try_import do
container_repository.migration_pre_import
end
end
before_transition %i[pre_importing import_aborted] => :pre_import_done do |container_repository|
container_repository.migration_pre_import_done_at = Time.zone.now
end
before_transition any => :importing do |container_repository|
container_repository.migration_import_started_at = Time.zone.now
container_repository.migration_import_done_at = nil
end
after_transition any => :importing do |container_repository|
container_repository.try_import do
container_repository.migration_import
end
end
before_transition %i[importing import_aborted] => :import_done do |container_repository|
container_repository.migration_import_done_at = Time.zone.now
end
before_transition any => :import_aborted do |container_repository|
container_repository.migration_aborted_in_state = container_repository.migration_state
container_repository.migration_aborted_at = Time.zone.now
container_repository.migration_retries_count += 1
end
before_transition import_aborted: any do |container_repository|
container_repository.migration_aborted_at = nil
container_repository.migration_aborted_in_state = nil
end
before_transition any => :import_skipped do |container_repository|
container_repository.migration_skipped_at = Time.zone.now
end
before_transition any => %i[import_done import_aborted] do |container_repository|
container_repository.run_after_commit do
::ContainerRegistry::Migration::EnqueuerWorker.perform_async
end
end
end
def self.exists_by_path?(path)
where(
project: path.repository_project,
name: path.repository_name
).exists?
end
def self.with_enabled_policy
joins('INNER JOIN container_expiration_policies ON container_repositories.project_id = container_expiration_policies.project_id')
.where(container_expiration_policies: { enabled: true })
end
def self.requiring_cleanup
with_enabled_policy
.where(container_repositories: { expiration_policy_cleanup_status: REQUIRING_CLEANUP_STATUSES })
.where('container_repositories.expiration_policy_started_at IS NULL OR container_repositories.expiration_policy_started_at < container_expiration_policies.next_run_at')
.where('container_expiration_policies.next_run_at < ?', Time.zone.now)
end
def self.with_unfinished_cleanup
with_enabled_policy.cleanup_unfinished
end
def self.with_stale_migration(before_timestamp)
stale_pre_importing = with_migration_states(:pre_importing)
.with_migration_pre_import_started_at_nil_or_before(before_timestamp)
stale_pre_import_done = with_migration_states(:pre_import_done)
.with_migration_pre_import_done_at_nil_or_before(before_timestamp)
stale_importing = with_migration_states(:importing)
.with_migration_import_started_at_nil_or_before(before_timestamp)
union = ::Gitlab::SQL::Union.new([
stale_pre_importing,
stale_pre_import_done,
stale_importing
])
from("(#{union.to_sql}) #{ContainerRepository.table_name}")
end
def self.with_target_import_tier
# overridden in ee
#
# Repositories are being migrated by tier on Saas, so we need to
# filter by plan/subscription which is not available in FOSS
all
end
def skip_import(reason:)
self.migration_skipped_reason = reason
super
end
def start_pre_import
return false unless ContainerRegistry::Migration.enabled?
super
end
def retry_pre_import
return false unless ContainerRegistry::Migration.enabled?
super
end
def retry_import
return false unless ContainerRegistry::Migration.enabled?
super
end
def finish_pre_import_and_start_import
# nothing to do between those two transitions for now.
finish_pre_import && start_import
end
def retry_aborted_migration
return unless migration_state == 'import_aborted'
case external_import_status
when 'native'
raise NativeImportError
when 'import_in_progress'
nil
when 'import_complete'
finish_import
when 'import_failed'
retry_import
when 'pre_import_in_progress'
nil
when 'pre_import_complete'
finish_pre_import_and_start_import
when 'pre_import_failed'
retry_pre_import
else
# If the import_status request fails, use the timestamp to guess current state
migration_pre_import_done_at ? retry_import : retry_pre_import
end
end
def try_import
raise ArgumentError, 'block not given' unless block_given?
try_count = 0
begin
try_count += 1
return true if yield == :ok
abort_import
false
rescue TooManyImportsError
if try_count <= ::ContainerRegistry::Migration.start_max_retries
sleep 0.1 * try_count
retry
else
abort_import
false
end
end
end
def last_import_step_done_at
[migration_pre_import_done_at, migration_import_done_at, migration_aborted_at].compact.max
end
def external_import_status
strong_memoize(:import_status) do
gitlab_api_client.import_status(self.path)
end
end
# rubocop: disable CodeReuse/ServiceClass
def registry
@registry ||= begin
token = Auth::ContainerRegistryAuthenticationService.full_access_token(path)
url = Gitlab.config.registry.api_url
host_port = Gitlab.config.registry.host_port
ContainerRegistry::Registry.new(url, token: token, path: host_port)
end
end
# rubocop: enable CodeReuse/ServiceClass
def path
@path ||= [project.full_path, name]
.select(&:present?).join('/').downcase
end
def location
File.join(registry.path, path)
end
def tag(tag)
ContainerRegistry::Tag.new(self, tag)
end
def manifest
@manifest ||= client.repository_tags(path)
end
def tags
return [] unless manifest && manifest['tags']
strong_memoize(:tags) do
manifest['tags'].sort.map do |tag|
ContainerRegistry::Tag.new(self, tag)
end
end
end
def tags_count
return 0 unless manifest && manifest['tags']
manifest['tags'].size
end
def blob(config)
ContainerRegistry::Blob.new(self, config)
end
def has_tags?
tags.any?
end
def root_repository?
name.empty?
end
def delete_tags!
return unless has_tags?
digests = tags.map { |tag| tag.digest }.compact.to_set
digests.map(&method(:delete_tag_by_digest)).all?
end
def delete_tag_by_digest(digest)
client.delete_repository_tag_by_digest(self.path, digest)
end
def delete_tag_by_name(name)
client.delete_repository_tag_by_name(self.path, name)
end
def reset_expiration_policy_started_at!
update!(expiration_policy_started_at: nil)
end
def start_expiration_policy!
update!(expiration_policy_started_at: Time.zone.now)
end
def migration_in_active_state?
migration_state.in?(ACTIVE_MIGRATION_STATES)
end
def migration_importing?
migration_state == 'importing'
end
def migration_pre_importing?
migration_state == 'pre_importing'
end
def migration_pre_import
return :error unless gitlab_api_client.supports_gitlab_api?
response = gitlab_api_client.pre_import_repository(self.path)
raise TooManyImportsError if response == :too_many_imports
response
end
def migration_import
return :error unless gitlab_api_client.supports_gitlab_api?
response = gitlab_api_client.import_repository(self.path)
raise TooManyImportsError if response == :too_many_imports
response
end
def self.build_from_path(path)
self.new(project: path.repository_project,
name: path.repository_name)
end
def self.find_or_create_from_path(path)
repository = safe_find_or_create_by(
project: path.repository_project,
name: path.repository_name
)
return repository if repository.persisted?
find_by_path!(path)
end
def self.build_root_repository(project)
self.new(project: project, name: '')
end
def self.find_by_path!(path)
self.find_by!(project: path.repository_project,
name: path.repository_name)
end
def self.find_by_path(path)
self.find_by(project: path.repository_project,
name: path.repository_name)
end
end
ContainerRepository.prepend_mod_with('ContainerRepository')
|