1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
# frozen_string_literal: true
module Gitlab
module Instrumentation
module RedisInterceptor
APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze
# These are temporary to help with investigating
# https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1183
DURATION_ERROR_THRESHOLD = 1.25.seconds
class MysteryRedisDurationError < StandardError
attr_reader :backtrace
def initialize(backtrace)
@backtrace = backtrace
end
end
def call(*args, &block)
start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined
start_real_time = Time.now
instrumentation_class.instance_count_request
instrumentation_class.redis_cluster_validate!(args.first)
super(*args, &block)
rescue ::Redis::BaseError => ex
instrumentation_class.instance_count_exception(ex)
raise ex
ensure
duration = Gitlab::Metrics::System.monotonic_time - start
unless APDEX_EXCLUDE.include?(command_from_args(args))
instrumentation_class.instance_observe_duration(duration)
end
if ::RequestStore.active?
# These metrics measure total Redis usage per Rails request / job.
instrumentation_class.increment_request_count
instrumentation_class.add_duration(duration)
instrumentation_class.add_call_details(duration, args)
end
if duration > DURATION_ERROR_THRESHOLD && Feature.enabled?(:report_on_long_redis_durations, default_enabled: :yaml)
Gitlab::ErrorTracking.track_exception(MysteryRedisDurationError.new(caller),
command: command_from_args(args),
duration: duration,
timestamp: start_real_time.iso8601(5))
end
end
def write(command)
measure_write_size(command) if ::RequestStore.active?
super
end
def read
result = super
measure_read_size(result) if ::RequestStore.active?
result
end
private
def measure_write_size(command)
size = 0
# Mimic what happens in
# https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8.
# This count is an approximation that omits the Redis protocol overhead
# of type prefixes, length prefixes and line endings.
command.each do |x|
size += begin
if x.is_a? Array
x.inject(0) { |sum, y| sum + y.to_s.bytesize }
else
x.to_s.bytesize
end
end
end
instrumentation_class.increment_write_bytes(size)
end
def measure_read_size(result)
# The Connection::Ruby#read class can return one of four types of results from read:
# https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406
#
# 1. Error (exception, will not reach this line)
# 2. Status (string)
# 3. Integer (will be converted to string by to_s.bytesize and thrown away)
# 4. "Binary" string (i.e. may contain zero byte)
# 5. Array of binary string
if result.is_a? Array
# Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here.
result.each { |x| measure_read_size(x) }
else
# This count is an approximation that omits the Redis protocol overhead
# of type prefixes, length prefixes and line endings.
instrumentation_class.increment_read_bytes(result.to_s.bytesize)
end
end
# That's required so it knows which GitLab Redis instance
# it's interacting with in order to categorize accordingly.
#
def instrumentation_class
@options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables
end
def command_from_args(args)
command = args[0]
command = command[0] if command.is_a?(Array)
command.to_s.downcase
end
end
end
end
|
