1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
# frozen_string_literal: true
module AlertManagement
# Module to support the processing of new alert payloads
# from various sources. Payloads may be for new alerts,
# existing alerts, or acting as a resolving alert.
#
# Performs processing-related tasks, such as creating system
# notes, creating or resolving related issues, and notifying
# stakeholders of the alert.
#
# Requires #project [Project] and #payload [Hash] methods
# to be defined.
module AlertProcessing
include BaseServiceUtility
include Gitlab::Utils::StrongMemoize
include ::IncidentManagement::Settings
# Updates or creates alert from payload for project
# including system notes
def process_alert
if alert.persisted?
process_existing_alert
else
process_new_alert
end
end
# Creates or closes issue for alert and notifies stakeholders
def complete_post_processing_tasks
process_incident_issues if process_issues?
send_alert_email if send_email?
end
def process_existing_alert
if resolving_alert?
process_resolved_alert
else
process_firing_alert
end
end
def process_resolved_alert
return unless auto_close_incident?
return close_issue(alert.issue) if alert.resolve(incoming_payload.ends_at)
logger.warn(
message: 'Unable to update AlertManagement::Alert status to resolved',
project_id: project.id,
alert_id: alert.id
)
end
def process_firing_alert
alert.register_new_event!
end
def close_issue(issue)
return if issue.blank? || issue.closed?
::Issues::CloseService
.new(project, User.alert_bot)
.execute(issue, system_note: false)
SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed?
end
def process_new_alert
if alert.save
alert.execute_services
SystemNoteService.create_new_alert(alert, alert_source)
else
logger.warn(
message: "Unable to create AlertManagement::Alert from #{alert_source}",
project_id: project.id,
alert_errors: alert.errors.messages
)
end
end
def process_incident_issues
return if alert.issue || alert.resolved?
::IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id)
end
def send_alert_email
notification_service
.async
.prometheus_alerts_fired(project, [alert])
end
def incoming_payload
strong_memoize(:incoming_payload) do
Gitlab::AlertManagement::Payload.parse(project, payload.to_h)
end
end
def alert
strong_memoize(:alert) do
find_existing_alert || build_new_alert
end
end
def find_existing_alert
return unless incoming_payload.gitlab_fingerprint
AlertManagement::Alert.not_resolved.for_fingerprint(project, incoming_payload.gitlab_fingerprint).first
end
def build_new_alert
AlertManagement::Alert.new(**incoming_payload.alert_params, ended_at: nil)
end
def resolving_alert?
incoming_payload.ends_at.present?
end
def alert_source
alert.monitoring_tool
end
def logger
@logger ||= Gitlab::AppLogger
end
end
end
|
