diff options
Diffstat (limited to 'app/services/projects/prometheus/alerts')
-rw-r--r-- | app/services/projects/prometheus/alerts/notify_service.rb | 31 |
1 files changed, 26 insertions, 5 deletions
diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index 56f65718d24..bc517ee3d6f 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -18,6 +18,14 @@ module Projects SUPPORTED_VERSION = '4' + # If feature flag :prometheus_notify_max_alerts is enabled truncate + # alerts to 100 and process only them. + # If feature flag is disabled process any amount of alerts. + # + # This is to mitigate incident: + # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6086 + PROCESS_MAX_ALERTS = 100 + def initialize(project, payload) @project = project @payload = payload @@ -28,6 +36,8 @@ module Projects return unprocessable_entity unless self.class.processable?(payload) return unauthorized unless valid_alert_manager_token?(token, integration) + truncate_alerts! if max_alerts_exceeded? + alert_responses = process_prometheus_alerts alert_response(alert_responses) @@ -49,12 +59,23 @@ module Projects Gitlab::Utils::DeepSize.new(payload).valid? end - def firings - @firings ||= alerts_by_status('firing') + def max_alerts_exceeded? + return false unless Feature.enabled?(:prometheus_notify_max_alerts, project, type: :ops) + + alerts.size > PROCESS_MAX_ALERTS end - def alerts_by_status(status) - alerts.select { |alert| alert['status'] == status } + def truncate_alerts! + Gitlab::AppLogger.warn( + message: 'Prometheus payload exceeded maximum amount of alerts. Truncating alerts.', + project_id: project.id, + alerts: { + total: alerts.size, + max: PROCESS_MAX_ALERTS + } + ) + + payload['alerts'] = alerts.first(PROCESS_MAX_ALERTS) end def alerts @@ -137,7 +158,7 @@ module Projects end def alert_response(alert_responses) - alerts = alert_responses.map { |resp| resp.payload[:alert] }.compact + alerts = alert_responses.flat_map { |resp| resp.payload[:alerts] }.compact success(alerts) end |