blob: e0e6d1418de2fe0af5216397ba6be31af29b2100 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
class StuckImportJobsWorker
include ApplicationWorker
include CronjobQueue
IMPORT_JOBS_EXPIRATION = 15.hours.to_i
def perform
projects_without_jid_count = mark_projects_without_jid_as_failed!
projects_with_jid_count = mark_projects_with_jid_as_failed!
Gitlab::Metrics.add_event(:stuck_import_jobs,
projects_without_jid_count: projects_without_jid_count,
projects_with_jid_count: projects_with_jid_count)
end
private
def mark_projects_without_jid_as_failed!
started_projects_without_jid.each do |project|
project.mark_import_as_failed(error_message)
end.count
end
def mark_projects_with_jid_as_failed!
completed_jids_count = 0
started_projects_with_jid.find_in_batches(batch_size: 500) do |group|
jids = group.map(&:import_jid)
# Find the jobs that aren't currently running or that exceeded the threshold.
completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set
if completed_jids.any?
completed_jids_count += completed_jids.count
group.each do |project|
project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid)
end
Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}")
end
end
completed_jids_count
end
def started_projects
Project.with_import_status(:started)
end
def started_projects_with_jid
started_projects.where.not(import_jid: nil)
end
def started_projects_without_jid
started_projects.where(import_jid: nil)
end
def error_message
"Import timed out. Import took longer than #{IMPORT_JOBS_EXPIRATION} seconds"
end
end
|
