diff options
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/all_queues.yml | 7 | ||||
-rw-r--r-- | app/workers/incident_management/process_prometheus_alert_worker.rb | 88 |
2 files changed, 95 insertions, 0 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index dd0eeaa9359..19158e7173c 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -528,6 +528,13 @@ :resource_boundary: :unknown :weight: 2 :idempotent: +- :name: incident_management:incident_management_process_prometheus_alert + :feature_category: :incident_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 2 + :idempotent: - :name: jira_importer:jira_import_advance_stage :feature_category: :importers :has_external_dependencies: diff --git a/app/workers/incident_management/process_prometheus_alert_worker.rb b/app/workers/incident_management/process_prometheus_alert_worker.rb new file mode 100644 index 00000000000..768e049c60e --- /dev/null +++ b/app/workers/incident_management/process_prometheus_alert_worker.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module IncidentManagement + class ProcessPrometheusAlertWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :incident_management + feature_category :incident_management + worker_resource_boundary :cpu + + def perform(project_id, alert_hash) + project = find_project(project_id) + return unless project + + parsed_alert = Gitlab::Alerting::Alert.new(project: project, payload: alert_hash) + event = find_prometheus_alert_event(parsed_alert) + + if event&.resolved? + issue = event.related_issues.order_created_at_desc.detect(&:opened?) + + close_issue(project, issue) + else + issue = create_issue(project, alert_hash) + + relate_issue_to_event(event, issue) + end + end + + private + + def find_project(project_id) + Project.find_by_id(project_id) + end + + def find_prometheus_alert_event(alert) + if alert.gitlab_managed? + find_gitlab_managed_event(alert) + else + find_self_managed_event(alert) + end + end + + def find_gitlab_managed_event(alert) + payload_key = payload_key_for_alert(alert) + + PrometheusAlertEvent.find_by_payload_key(payload_key) + end + + def find_self_managed_event(alert) + payload_key = payload_key_for_alert(alert) + + SelfManagedPrometheusAlertEvent.find_by_payload_key(payload_key) + end + + def payload_key_for_alert(alert) + if alert.gitlab_managed? + PrometheusAlertEvent.payload_key_for(alert.metric_id, alert.starts_at_raw) + else + SelfManagedPrometheusAlertEvent.payload_key_for(alert.starts_at_raw, alert.title, alert.full_query) + end + end + + def create_issue(project, alert) + IncidentManagement::CreateIssueService + .new(project, alert) + .execute + .dig(:issue) + end + + def close_issue(project, issue) + return if issue.blank? || issue.closed? + + processed_issue = Issues::CloseService + .new(project, User.alert_bot) + .execute(issue, system_note: false) + + SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if processed_issue.reset.closed? + end + + def relate_issue_to_event(event, issue) + return unless event && issue + + if event.related_issues.exclude?(issue) + event.related_issues << issue + end + end + end +end |