1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
# frozen_string_literal: true
require 'redis'
module Gitlab
module Instrumentation
class RedisBase
class << self
include ::Gitlab::Utils::StrongMemoize
include ::Gitlab::Instrumentation::RedisPayload
InstrumentationStorage = ::Gitlab::Instrumentation::Storage
# TODO: To be used by https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/395
# as a 'label' alias.
def storage_key
self.name.demodulize.underscore
end
def add_duration(duration)
return unless InstrumentationStorage.active?
InstrumentationStorage[call_duration_key] ||= 0
InstrumentationStorage[call_duration_key] += duration
end
def add_call_details(duration, commands)
return unless Gitlab::PerformanceBar.enabled_for_request?
detail_store << {
commands: commands,
duration: duration,
backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
}
end
def increment_request_count(amount = 1)
return unless InstrumentationStorage.active?
InstrumentationStorage[request_count_key] ||= 0
InstrumentationStorage[request_count_key] += amount
end
def increment_read_bytes(num_bytes)
return unless InstrumentationStorage.active?
InstrumentationStorage[read_bytes_key] ||= 0
InstrumentationStorage[read_bytes_key] += num_bytes
end
def increment_write_bytes(num_bytes)
return unless InstrumentationStorage.active?
InstrumentationStorage[write_bytes_key] ||= 0
InstrumentationStorage[write_bytes_key] += num_bytes
end
def increment_cross_slot_request_count(amount = 1)
return unless InstrumentationStorage.active?
InstrumentationStorage[cross_slots_key] ||= 0
InstrumentationStorage[cross_slots_key] += amount
end
def increment_allowed_cross_slot_request_count(amount = 1)
return unless InstrumentationStorage.active?
InstrumentationStorage[allowed_cross_slots_key] ||= 0
InstrumentationStorage[allowed_cross_slots_key] += amount
end
def get_request_count
InstrumentationStorage[request_count_key] || 0
end
def read_bytes
InstrumentationStorage[read_bytes_key] || 0
end
def write_bytes
InstrumentationStorage[write_bytes_key] || 0
end
def detail_store
InstrumentationStorage[call_details_key] ||= []
end
def get_cross_slot_request_count
InstrumentationStorage[cross_slots_key] || 0
end
def get_allowed_cross_slot_request_count
InstrumentationStorage[allowed_cross_slots_key] || 0
end
def query_time
query_time = InstrumentationStorage[call_duration_key] || 0
query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
end
def redis_cluster_validate!(commands)
return true unless @redis_cluster_validation
result = ::Gitlab::Instrumentation::RedisClusterValidator.validate(commands)
return true if result.nil?
if !result[:valid] && !result[:allowed] && (Rails.env.development? || Rails.env.test?)
raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands"
end
increment_allowed_cross_slot_request_count if result[:allowed]
result[:valid]
end
def enable_redis_cluster_validation
@redis_cluster_validation = true
self
end
def instance_count_request(amount = 1)
@request_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_requests_total, 'Client side Redis request count, per Redis server')
@request_counter.increment({ storage: storage_key }, amount)
end
def instance_count_exception(ex)
# This metric is meant to give a client side view of how the Redis
# server is doing. Redis itself does not expose error counts. This
# metric can be used for Redis alerting and service health monitoring.
@exception_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_exceptions_total, 'Client side Redis exception count, per Redis server, per exception class')
@exception_counter.increment({ storage: storage_key, exception: ex.class.to_s })
end
def instance_count_cluster_redirection(ex)
# This metric is meant to give a client side view of how often are commands
# redirected to the right node, especially during resharding..
# This metric can be used for Redis alerting and service health monitoring.
@redirection_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_redirections_total, 'Client side Redis Cluster redirection count, per Redis node, per slot')
@redirection_counter.increment(decompose_redirection_message(ex.message).merge({ storage: storage_key }))
end
def instance_observe_duration(duration)
@request_latency_histogram ||= Gitlab::Metrics.histogram(
:gitlab_redis_client_requests_duration_seconds,
'Client side Redis request latency, per Redis server, excluding blocking commands',
{},
[0.1, 0.5, 0.75, 1]
)
@request_latency_histogram.observe({ storage: storage_key }, duration)
end
def log_exception(ex)
::Gitlab::ErrorTracking.log_exception(ex, storage: storage_key)
end
private
def request_count_key
strong_memoize(:request_count_key) { build_key(:redis_request_count) }
end
def read_bytes_key
strong_memoize(:read_bytes_key) { build_key(:redis_read_bytes) }
end
def write_bytes_key
strong_memoize(:write_bytes_key) { build_key(:redis_write_bytes) }
end
def call_duration_key
strong_memoize(:call_duration_key) { build_key(:redis_call_duration) }
end
def call_details_key
strong_memoize(:call_details_key) { build_key(:redis_call_details) }
end
def cross_slots_key
strong_memoize(:cross_slots_key) { build_key(:redis_cross_slot_request_count) }
end
def allowed_cross_slots_key
strong_memoize(:allowed_cross_slots_key) { build_key(:redis_allowed_cross_slot_request_count) }
end
def build_key(namespace)
"#{storage_key}_#{namespace}"
end
def decompose_redirection_message(err_msg)
redirection_type, _, target_node_key = err_msg.split
{ redirection_type: redirection_type, target_node_key: target_node_key }
end
end
end
end
end
|
