1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
|
# frozen_string_literal: true
require_relative '../../../lib/gitlab/process_supervisor'
RSpec.describe Gitlab::ProcessSupervisor, feature_category: :application_performance do
let(:health_check_interval_seconds) { 0.1 }
let(:check_terminate_interval_seconds) { 1 }
let(:forwarded_signals) { [] }
let(:term_signals) { [] }
let(:process_ids) { [spawn_process, spawn_process] }
def spawn_process
Process.spawn('while true; do sleep 1; done').tap do |pid|
Process.detach(pid)
end
end
subject(:supervisor) do
described_class.new(
health_check_interval_seconds: health_check_interval_seconds,
check_terminate_interval_seconds: check_terminate_interval_seconds,
terminate_timeout_seconds: 1 + check_terminate_interval_seconds,
forwarded_signals: forwarded_signals,
term_signals: term_signals
)
end
after do
process_ids.each do |pid|
Process.kill('KILL', pid)
rescue Errno::ESRCH
# Ignore if a process wasn't actually alive.
end
supervisor.stop
end
describe '#supervise' do
context 'while supervised processes are alive' do
it 'does not invoke callback' do
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
pids_killed = []
supervisor.supervise(process_ids) do |dead_pids|
pids_killed = dead_pids
[]
end
# Wait several times the poll frequency of the supervisor.
sleep health_check_interval_seconds * 10
expect(pids_killed).to be_empty
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
end
end
context 'when a supervised process dies' do
it 'triggers callback with the dead PIDs and adds new PIDs to supervised PIDs' do
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
pids_killed = []
supervisor.supervise(process_ids) do |dead_pids|
pids_killed = dead_pids
[42] # Fake starting a new process in place of the terminated one.
end
# Terminate a supervised process.
Process.kill('TERM', process_ids.first)
await_condition(sleep_sec: health_check_interval_seconds) do
pids_killed == [process_ids.first]
end
expect(Gitlab::ProcessManagement.process_alive?(process_ids.first)).to be(false)
expect(Gitlab::ProcessManagement.process_alive?(process_ids.last)).to be(true)
expect(supervisor.supervised_pids).to match_array([process_ids.last, 42])
end
it 'deduplicates PIDs returned from callback' do
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
pids_killed = []
supervisor.supervise(process_ids) do |dead_pids|
pids_killed = dead_pids
# Fake a new process having the same pid as one that was just terminated.
[process_ids.last]
end
# Terminate a supervised process.
Process.kill('TERM', process_ids.first)
await_condition(sleep_sec: health_check_interval_seconds) do
pids_killed == [process_ids.first]
end
expect(supervisor.supervised_pids).to contain_exactly(process_ids.last)
end
it 'accepts single PID returned from callback' do
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
pids_killed = []
supervisor.supervise(process_ids) do |dead_pids|
pids_killed = dead_pids
42
end
# Terminate a supervised process.
Process.kill('TERM', process_ids.first)
await_condition(sleep_sec: health_check_interval_seconds) do
pids_killed == [process_ids.first]
end
expect(supervisor.supervised_pids).to contain_exactly(42, process_ids.last)
end
context 'but supervisor has entered shutdown' do
it 'does not trigger callback again' do
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
callback_count = 0
supervisor.supervise(process_ids) do |dead_pids|
callback_count += 1
Thread.new { supervisor.shutdown }
[42]
end
# Terminate the supervised processes to trigger more than 1 callback.
Process.kill('TERM', process_ids.first)
Process.kill('TERM', process_ids.last)
await_condition(sleep_sec: health_check_interval_seconds * 3) do
supervisor.alive == false
end
# Since we shut down the supervisor during the first callback, it should not
# be called anymore.
expect(callback_count).to eq(1)
end
end
end
context 'signal handling' do
before do
allow(supervisor).to receive(:sleep)
allow(Gitlab::ProcessManagement).to receive(:trap_signals)
allow(Gitlab::ProcessManagement).to receive(:all_alive?).and_return(false)
allow(Gitlab::ProcessManagement).to receive(:signal_processes).with(process_ids, anything)
end
context 'termination signals' do
let(:term_signals) { %i(INT TERM) }
context 'when TERM results in timely shutdown of processes' do
it 'forwards them to observed processes without waiting for grace period to expire' do
allow(Gitlab::ProcessManagement).to receive(:any_alive?).and_return(false)
expect(Gitlab::ProcessManagement).to receive(:trap_signals).ordered.with(%i(INT TERM)).and_yield(:TERM)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM)
expect(supervisor).not_to receive(:sleep).with(check_terminate_interval_seconds)
supervisor.supervise(process_ids) { [] }
end
end
context 'when TERM does not result in timely shutdown of processes' do
it 'issues a KILL signal after the grace period expires' do
expect(Gitlab::ProcessManagement).to receive(:trap_signals).with(%i(INT TERM)).and_yield(:TERM)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM)
expect(supervisor).to receive(:sleep).ordered.with(check_terminate_interval_seconds).at_least(:once)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, '-KILL')
supervisor.supervise(process_ids) { [] }
end
end
end
context 'forwarded signals' do
let(:forwarded_signals) { %i(USR1) }
it 'forwards given signals to the observed processes' do
expect(Gitlab::ProcessManagement).to receive(:trap_signals).with(%i(USR1)).and_yield(:USR1)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :USR1)
supervisor.supervise(process_ids) { [] }
end
end
end
end
describe '#shutdown' do
context 'when supervisor is supervising processes' do
before do
supervisor.supervise(process_ids)
end
context 'when supervisor is alive' do
it 'signals TERM then KILL to all supervised processes' do
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, '-KILL')
supervisor.shutdown
end
it 'stops the supervisor' do
expect { supervisor.shutdown }.to change { supervisor.alive }.from(true).to(false)
end
end
context 'when supervisor has already shut down' do
before do
supervisor.shutdown
end
it 'does nothing' do
expect(supervisor.alive).to be(false)
expect(Gitlab::ProcessManagement).not_to receive(:signal_processes)
supervisor.shutdown
end
end
end
context 'when supervisor never started' do
it 'does nothing' do
expect(supervisor.alive).to be(false)
expect(Gitlab::ProcessManagement).not_to receive(:signal_processes)
supervisor.shutdown
end
end
end
def await_condition(timeout_sec: 5, sleep_sec: 0.1)
Timeout.timeout(timeout_sec) do
sleep sleep_sec until yield
end
end
end
|