blob: 88a8f98aaf6652608dc1850e456d037c630c57ff (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
# frozen_string_literal: true
module RepositoryCheck
class BatchWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
sidekiq_options retry: 3
include RepositoryCheckQueue
include ExclusiveLeaseGuard
RUN_TIME = 3600
BATCH_SIZE = 10_000
LEASE_TIMEOUT = 1.hour
attr_reader :shard_name
loggable_arguments 0
def perform(shard_name)
@shard_name = shard_name
return unless Gitlab::CurrentSettings.repository_checks_enabled
return unless Gitlab::ShardHealthCache.healthy_shard?(shard_name)
try_obtain_lease do
perform_repository_checks
end
end
def lease_timeout
LEASE_TIMEOUT
end
def lease_key
"repository_check_batch_worker:#{shard_name}"
end
def perform_repository_checks
start = Time.current
# This loop will break after a little more than one hour ('a little
# more' because `git fsck` may take a few minutes), or if it runs out of
# projects to check. By default sidekiq-cron will start a new
# RepositoryCheckWorker each hour so that as long as there are repositories to
# check, only one (or two) will be checked at a time.
project_ids.each do |project_id|
break if Time.current - start >= RUN_TIME
next unless try_obtain_lease_for_project(project_id)
SingleRepositoryWorker.new.perform(project_id)
end
end
private
# Project.find_each does not support WHERE clauses and
# Project.find_in_batches does not support ordering. So we just build an
# array of ID's. This is OK because we do it only once an hour, because
# getting ID's from Postgres is not terribly slow, and because no user
# has to sit and wait for this query to finish.
def project_ids
never_checked_project_ids(BATCH_SIZE) + old_checked_project_ids(BATCH_SIZE)
end
# rubocop: disable CodeReuse/ActiveRecord
def never_checked_project_ids(batch_size)
projects_on_shard.where(last_repository_check_at: nil)
.where('created_at < ?', 24.hours.ago)
.limit(batch_size).pluck(:id)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def old_checked_project_ids(batch_size)
projects_on_shard.where.not(last_repository_check_at: nil)
.where('last_repository_check_at < ?', 1.month.ago)
.reorder(last_repository_check_at: :asc)
.limit(batch_size).pluck(:id)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def projects_on_shard
Project.where(repository_storage: shard_name)
end
# rubocop: enable CodeReuse/ActiveRecord
def try_obtain_lease_for_project(id)
# Use a 24-hour timeout because on servers/projects where 'git fsck' is
# super slow we definitely do not want to run it twice in parallel.
Gitlab::ExclusiveLease.new(
"project_repository_check:#{id}",
timeout: 24.hours
).try_obtain
end
end
end
RepositoryCheck::BatchWorker.prepend_mod_with('RepositoryCheck::BatchWorker')
|
