diff options
Diffstat (limited to 'app/services/web_hook_service.rb')
-rw-r--r-- | app/services/web_hook_service.rb | 64 |
1 files changed, 63 insertions, 1 deletions
diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index 5a51b42f9f9..654d9356739 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -6,6 +6,18 @@ class WebHookService attr_reader :body, :headers, :code + def success? + false + end + + def redirection? + false + end + + def internal_server_error? + true + end + def initialize @headers = Gitlab::HTTP::Response::Headers.new({}) @body = '' @@ -15,6 +27,7 @@ class WebHookService REQUEST_BODY_SIZE_LIMIT = 25.megabytes GITLAB_EVENT_HEADER = 'X-Gitlab-Event' + MAX_FAILURES = 100 attr_accessor :hook, :data, :hook_name, :request_options @@ -33,6 +46,8 @@ class WebHookService end def execute + return { status: :error, message: 'Hook disabled' } unless hook.executable? + start_time = Gitlab::Metrics::System.monotonic_time response = if parsed_url.userinfo.blank? @@ -76,7 +91,11 @@ class WebHookService end def async_execute - WebHookWorker.perform_async(hook.id, data, hook_name) + if rate_limited?(hook) + log_rate_limit(hook) + else + WebHookWorker.perform_async(hook.id, data, hook_name) + end end private @@ -104,6 +123,8 @@ class WebHookService end def log_execution(trigger:, url:, request_data:, response:, execution_duration:, error_message: nil) + handle_failure(response, hook) + WebHookLog.create( web_hook: hook, trigger: trigger, @@ -118,6 +139,17 @@ class WebHookService ) end + def handle_failure(response, hook) + if response.success? || response.redirection? + hook.enable! + elsif response.internal_server_error? + next_backoff = hook.next_backoff + hook.update!(disabled_until: next_backoff.from_now, backoff_count: hook.backoff_count + 1) + else + hook.update!(recent_failures: hook.recent_failures + 1) if hook.recent_failures < MAX_FAILURES + end + end + def build_headers(hook_name) @headers ||= begin { @@ -142,4 +174,34 @@ class WebHookService response.body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '') end + + def rate_limited?(hook) + return false unless Feature.enabled?(:web_hooks_rate_limit, default_enabled: :yaml) + return false if rate_limit.nil? + + Gitlab::ApplicationRateLimiter.throttled?( + :web_hook_calls, + scope: [hook], + threshold: rate_limit + ) + end + + def rate_limit + @rate_limit ||= hook.rate_limit + end + + def log_rate_limit(hook) + payload = { + message: 'Webhook rate limit exceeded', + hook_id: hook.id, + hook_type: hook.type, + hook_name: hook_name + } + + Gitlab::AuthLogger.error(payload) + + # Also log into application log for now, so we can use this information + # to determine suitable limits for gitlab.com + Gitlab::AppLogger.error(payload) + end end |