From 4226aca420920c1844e8eade4798a2dff188a6fc Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 17 Oct 2019 09:07:07 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- .../gitlab/sidekiq_daemon/memory_killer_spec.rb | 173 ++++++++++++++++----- 1 file changed, 135 insertions(+), 38 deletions(-) (limited to 'spec/lib/gitlab/sidekiq_daemon') diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb index adea2ba30b3..45bcc71dfcb 100644 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -7,9 +7,12 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do let(:pid) { 12345 } before do - allow(memory_killer).to receive(:pid).and_return(pid) allow(Sidekiq.logger).to receive(:info) allow(Sidekiq.logger).to receive(:warn) + allow(memory_killer).to receive(:pid).and_return(pid) + + # make sleep no-op + allow(memory_killer).to receive(:sleep) {} end describe '#run_thread' do @@ -39,8 +42,9 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do pid: pid, message: "Exception from run_thread: My Exception") - expect(memory_killer).to receive(:rss_within_range?).twice.and_raise(StandardError, 'My Exception') - expect(memory_killer).to receive(:sleep).twice.with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS) + expect(memory_killer).to receive(:rss_within_range?) + .twice + .and_raise(StandardError, 'My Exception') expect { subject }.not_to raise_exception end @@ -52,7 +56,9 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do pid: pid, message: "Exception from run_thread: My Exception") - expect(memory_killer).to receive(:rss_within_range?).once.and_raise(Exception, 'My Exception') + expect(memory_killer).to receive(:rss_within_range?) + .once + .and_raise(Exception, 'My Exception') expect(memory_killer).to receive(:sleep).with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS) expect(Sidekiq.logger).to receive(:warn).once @@ -78,7 +84,9 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'not invoke restart_sidekiq when rss in range' do - expect(memory_killer).to receive(:rss_within_range?).twice.and_return(true) + expect(memory_killer).to receive(:rss_within_range?) + .twice + .and_return(true) expect(memory_killer).not_to receive(:restart_sidekiq) @@ -86,9 +94,12 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'invoke restart_sidekiq when rss not in range' do - expect(memory_killer).to receive(:rss_within_range?).at_least(:once).and_return(false) + expect(memory_killer).to receive(:rss_within_range?) + .at_least(:once) + .and_return(false) - expect(memory_killer).to receive(:restart_sidekiq).at_least(:once) + expect(memory_killer).to receive(:restart_sidekiq) + .at_least(:once) subject end @@ -97,10 +108,9 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do describe '#stop_working' do subject { memory_killer.send(:stop_working)} - it 'changed enable? to false' do - expect(memory_killer.send(:enabled?)).to be true - subject - expect(memory_killer.send(:enabled?)).to be false + it 'changes enable? to false' do + expect { subject }.to change { memory_killer.send(:enabled?) } + .from(true).to(false) end end @@ -121,8 +131,12 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do it 'return true when everything is within limit' do expect(memory_killer).to receive(:get_rss).and_return(100) - expect(memory_killer).to receive(:soft_limit_rss).and_return(200) - expect(memory_killer).to receive(:hard_limit_rss).and_return(300) + expect(memory_killer).to receive(:get_soft_limit_rss).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss).and_return(300) + + expect(memory_killer).to receive(:refresh_state) + .with(:running) + .and_call_original expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original expect(memory_killer).not_to receive(:log_rss_out_of_range) @@ -131,9 +145,17 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return false when rss exceeds hard_limit_rss' do - expect(memory_killer).to receive(:get_rss).and_return(400) - expect(memory_killer).to receive(:soft_limit_rss).at_least(:once).and_return(200) - expect(memory_killer).to receive(:hard_limit_rss).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_rss).at_least(:once).and_return(400) + expect(memory_killer).to receive(:get_soft_limit_rss).at_least(:once).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss).at_least(:once).and_return(300) + + expect(memory_killer).to receive(:refresh_state) + .with(:running) + .and_call_original + + expect(memory_killer).to receive(:refresh_state) + .with(:above_soft_limit) + .and_call_original expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original @@ -143,13 +165,21 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return false when rss exceed hard_limit_rss after a while' do - expect(memory_killer).to receive(:get_rss).and_return(250, 400) - expect(memory_killer).to receive(:soft_limit_rss).at_least(:once).and_return(200) - expect(memory_killer).to receive(:hard_limit_rss).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_rss).and_return(250, 400, 400) + expect(memory_killer).to receive(:get_soft_limit_rss).at_least(:once).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss).at_least(:once).and_return(300) + + expect(memory_killer).to receive(:refresh_state) + .with(:running) + .and_call_original + + expect(memory_killer).to receive(:refresh_state) + .at_least(:once) + .with(:above_soft_limit) + .and_call_original expect(Gitlab::Metrics::System).to receive(:monotonic_time).twice.and_call_original expect(memory_killer).to receive(:sleep).with(check_interval_seconds) - expect(memory_killer).to receive(:log_rss_out_of_range).with(400, 300, 200) expect(subject).to be false @@ -157,8 +187,16 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do it 'return true when rss below soft_limit_rss after a while within GRACE_BALLOON_SECONDS' do expect(memory_killer).to receive(:get_rss).and_return(250, 100) - expect(memory_killer).to receive(:soft_limit_rss).and_return(200, 200) - expect(memory_killer).to receive(:hard_limit_rss).and_return(300, 300) + expect(memory_killer).to receive(:get_soft_limit_rss).and_return(200, 200) + expect(memory_killer).to receive(:get_hard_limit_rss).and_return(300, 300) + + expect(memory_killer).to receive(:refresh_state) + .with(:running) + .and_call_original + + expect(memory_killer).to receive(:refresh_state) + .with(:above_soft_limit) + .and_call_original expect(Gitlab::Metrics::System).to receive(:monotonic_time).twice.and_call_original expect(memory_killer).to receive(:sleep).with(check_interval_seconds) @@ -168,17 +206,27 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do expect(subject).to be true end - it 'return false when rss exceed soft_limit_rss longer than GRACE_BALLOON_SECONDS' do - expect(memory_killer).to receive(:get_rss).exactly(4).times.and_return(250) - expect(memory_killer).to receive(:soft_limit_rss).exactly(5).times.and_return(200) - expect(memory_killer).to receive(:hard_limit_rss).exactly(5).times.and_return(300) + context 'when exceeding GRACE_BALLOON_SECONDS' do + let(:grace_balloon_seconds) { 0 } - expect(Gitlab::Metrics::System).to receive(:monotonic_time).exactly(5).times.and_call_original - expect(memory_killer).to receive(:sleep).exactly(3).times.with(check_interval_seconds).and_call_original + it 'return false when rss exceed soft_limit_rss' do + allow(memory_killer).to receive(:get_rss).and_return(250) + allow(memory_killer).to receive(:get_soft_limit_rss).and_return(200) + allow(memory_killer).to receive(:get_hard_limit_rss).and_return(300) - expect(memory_killer).to receive(:log_rss_out_of_range).with(250, 300, 200) + expect(memory_killer).to receive(:refresh_state) + .with(:running) + .and_call_original - expect(subject).to be false + expect(memory_killer).to receive(:refresh_state) + .with(:above_soft_limit) + .and_call_original + + expect(memory_killer).to receive(:log_rss_out_of_range) + .with(250, 300, 200) + + expect(subject).to be false + end end end @@ -190,19 +238,42 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do before do stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) allow(Sidekiq).to receive(:options).and_return(timeout: 9) + allow(memory_killer).to receive(:get_rss).and_return(100) + allow(memory_killer).to receive(:get_soft_limit_rss).and_return(200) + allow(memory_killer).to receive(:get_hard_limit_rss).and_return(300) end it 'send signal' do - expect(memory_killer).to receive(:signal_and_wait).with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs').ordered - expect(memory_killer).to receive(:signal_and_wait).with(11, 'SIGTERM', 'gracefully shut down').ordered - expect(memory_killer).to receive(:signal_pgroup).with('SIGKILL', 'die').ordered + expect(memory_killer).to receive(:refresh_state) + .with(:stop_fetching_new_jobs) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_and_wait) + .with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs') + .ordered + + expect(memory_killer).to receive(:refresh_state) + .with(:shutting_down) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_and_wait) + .with(11, 'SIGTERM', 'gracefully shut down') + .ordered + + expect(memory_killer).to receive(:refresh_state) + .with(:killing_sidekiq) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_pgroup) + .with('SIGKILL', 'die') + .ordered subject end end describe '#signal_and_wait' do - let(:time) { 7 } + let(:time) { 0 } let(:signal) { 'my-signal' } let(:explanation) { 'my-explanation' } let(:check_interval_seconds) { 2 } @@ -226,14 +297,17 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'send signal and wait till deadline if any job not finished' do - expect(Process).to receive(:kill).with(signal, pid).ordered - expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original.at_least(:once) + expect(Process).to receive(:kill) + .with(signal, pid) + .ordered + + expect(Gitlab::Metrics::System).to receive(:monotonic_time) + .and_call_original + .at_least(:once) expect(memory_killer).to receive(:enabled?).and_return(true).at_least(:once) expect(memory_killer).to receive(:any_jobs?).and_return(true).at_least(:once) - expect(memory_killer).to receive(:sleep).and_call_original.exactly(4).times - subject end end @@ -401,4 +475,27 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do expect(subject).to eq(10) end end + + describe '#refresh_state' do + let(:metrics) { memory_killer.instance_variable_get(:@metrics) } + + subject { memory_killer.send(:refresh_state, :shutting_down) } + + it 'calls gitlab metrics gauge set methods' do + expect(memory_killer).to receive(:get_rss) { 1010 } + expect(memory_killer).to receive(:get_soft_limit_rss) { 1020 } + expect(memory_killer).to receive(:get_hard_limit_rss) { 1040 } + + expect(metrics[:sidekiq_memory_killer_phase]).to receive(:set) + .with({}, described_class::PHASE[:shutting_down]) + expect(metrics[:sidekiq_current_rss]).to receive(:set) + .with({}, 1010) + expect(metrics[:sidekiq_memory_killer_soft_limit_rss]).to receive(:set) + .with({}, 1020) + expect(metrics[:sidekiq_memory_killer_hard_limit_rss]).to receive(:set) + .with({}, 1040) + + subject + end + end end -- cgit v1.2.1