Limit Sidekiq args logging to 10 KB of JSON

When logging arguments from Sidekiq to JSON, restrict the size of `args`
to 10 KB (when converted to JSON). This is to avoid blowing up with
excessively large job payloads.

1 files changed, 17 insertions, 0 deletions

diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index e86db8db3a1..fdc0d518c59 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -5,6 +5,7 @@ module Gitlab
     class StructuredLogger
       START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze
       DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze
+      MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes
 
       def call(job, queue)
         started_at = current_time
@@ -64,6 +65,7 @@ module Gitlab
         job['pid'] = ::Process.pid
 
         job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
+        job['args'] = limited_job_args(job['args']) if job['args']
 
         convert_to_iso8601(job, START_TIMESTAMP_FIELDS)
 
@@ -93,6 +95,21 @@ module Gitlab
 
         Time.at(timestamp).utc.iso8601(3)
       end
+
+      def limited_job_args(args)
+        return unless args.is_a?(Array)
+
+        total_length = 0
+        limited_args = args.take_while do |arg|
+          total_length += arg.to_json.length
+
+          total_length <= MAXIMUM_JOB_ARGUMENTS_LENGTH
+        end
+
+        limited_args.push('...') if total_length > MAXIMUM_JOB_ARGUMENTS_LENGTH
+
+        limited_args
+      end
     end
   end
 end