diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-18 18:06:14 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-18 18:06:14 +0000 |
commit | b08279013423a66f06f5edde4e067f328fe135bd (patch) | |
tree | 47aca1a9b0655cd7861bddf31ec17d4b302fc4ea /spec/lib/gitlab | |
parent | 4584eb0e07d372d6014de16ab359965475184c99 (diff) | |
download | gitlab-ce-b08279013423a66f06f5edde4e067f328fe135bd.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'spec/lib/gitlab')
-rw-r--r-- | spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb | 408 | ||||
-rw-r--r-- | spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb | 48 | ||||
-rw-r--r-- | spec/lib/gitlab/sidekiq_middleware/monitor_spec.rb | 2 | ||||
-rw-r--r-- | spec/lib/gitlab/usage_data_spec.rb | 2 |
4 files changed, 453 insertions, 7 deletions
diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb new file mode 100644 index 00000000000..3b3fcfc89d7 --- /dev/null +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -0,0 +1,408 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::SidekiqDaemon::MemoryKiller do + let(:memory_killer) { described_class.new } + let(:pid) { 12345 } + + before do + allow(memory_killer).to receive(:pid).and_return(pid) + allow(Sidekiq.logger).to receive(:info) + allow(Sidekiq.logger).to receive(:warn) + end + + describe '#start_working' do + subject { memory_killer.send(:start_working) } + + before do + # let enabled? return 3 times: true, true, false + allow(memory_killer).to receive(:enabled?).and_return(true, true, false) + end + + context 'when structured logging is used' do + it 'logs start message once' do + expect(Sidekiq.logger).to receive(:info).once + .with( + class: described_class.to_s, + action: 'start', + pid: pid, + message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon') + + subject + end + + it 'logs StandardError message twice' do + expect(Sidekiq.logger).to receive(:warn).twice + .with( + class: described_class.to_s, + pid: pid, + message: "Exception from start_working: My Exception") + + expect(memory_killer).to receive(:rss_within_range?).twice.and_raise(StandardError, 'My Exception') + + expect { subject }.not_to raise_exception + end + + it 'logs exception message once and raise execption and log stop message' do + expect(Sidekiq.logger).to receive(:warn).once + .with( + class: described_class.to_s, + pid: pid, + message: "Exception from start_working: My Exception") + + expect(memory_killer).to receive(:rss_within_range?).once.and_raise(Exception, 'My Exception') + + expect(Sidekiq.logger).to receive(:warn).once + .with( + class: described_class.to_s, + action: 'stop', + pid: pid, + message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon') + + expect { subject }.to raise_exception + end + + it 'logs stop message once' do + expect(Sidekiq.logger).to receive(:warn).once + .with( + class: described_class.to_s, + action: 'stop', + pid: pid, + message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon') + + subject + end + end + + it 'invoke rss_within_range? twice' do + expect(memory_killer).to receive(:rss_within_range?).twice + + subject + end + + it 'not invoke restart_sidekiq when rss in range' do + expect(memory_killer).to receive(:rss_within_range?).twice.and_return(true) + + expect(memory_killer).not_to receive(:restart_sidekiq) + + subject + end + + it 'invoke restart_sidekiq when rss not in range' do + expect(memory_killer).to receive(:rss_within_range?).at_least(:once).and_return(false) + + expect(memory_killer).to receive(:restart_sidekiq).at_least(:once) + + subject + end + end + + describe '#stop_working' do + subject { memory_killer.send(:stop_working)} + + it 'changed enable? to false' do + expect(memory_killer.send(:enabled?)).to be true + subject + expect(memory_killer.send(:enabled?)).to be false + end + end + + describe '#rss_within_range?' do + let(:shutdown_timeout_seconds) { 7 } + let(:check_interval_seconds) { 2 } + let(:grace_balloon_seconds) { 5 } + + subject { memory_killer.send(:rss_within_range?) } + + before do + stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) + stub_const("#{described_class}::CHECK_INTERVAL_SECONDS", check_interval_seconds) + stub_const("#{described_class}::GRACE_BALLOON_SECONDS", grace_balloon_seconds) + allow(Process).to receive(:getpgrp).and_return(pid) + allow(Sidekiq).to receive(:options).and_return(timeout: 9) + end + + it 'return true when everything is within limit' do + expect(memory_killer).to receive(:get_rss).and_return(100) + expect(memory_killer).to receive(:soft_limit_rss).and_return(200) + expect(memory_killer).to receive(:hard_limit_rss).and_return(300) + + expect(Time).to receive(:now).and_call_original + expect(memory_killer).not_to receive(:log_rss_out_of_range) + + expect(subject).to be true + end + + it 'return false when rss exceeds hard_limit_rss' do + expect(memory_killer).to receive(:get_rss).and_return(400) + expect(memory_killer).to receive(:soft_limit_rss).at_least(:once).and_return(200) + expect(memory_killer).to receive(:hard_limit_rss).at_least(:once).and_return(300) + + expect(Time).to receive(:now).and_call_original + + expect(memory_killer).to receive(:log_rss_out_of_range).with(400, 300, 200) + + expect(subject).to be false + end + + it 'return false when rss exceed hard_limit_rss after a while' do + expect(memory_killer).to receive(:get_rss).and_return(250, 400) + expect(memory_killer).to receive(:soft_limit_rss).at_least(:once).and_return(200) + expect(memory_killer).to receive(:hard_limit_rss).at_least(:once).and_return(300) + + expect(Time).to receive(:now).twice.and_call_original + expect(memory_killer).to receive(:sleep).with(check_interval_seconds) + + expect(memory_killer).to receive(:log_rss_out_of_range).with(400, 300, 200) + + expect(subject).to be false + end + + it 'return true when rss below soft_limit_rss after a while within GRACE_BALLOON_SECONDS' do + expect(memory_killer).to receive(:get_rss).and_return(250, 100) + expect(memory_killer).to receive(:soft_limit_rss).and_return(200, 200) + expect(memory_killer).to receive(:hard_limit_rss).and_return(300, 300) + + expect(Time).to receive(:now).twice.and_call_original + expect(memory_killer).to receive(:sleep).with(check_interval_seconds) + + expect(memory_killer).not_to receive(:log_rss_out_of_range) + + expect(subject).to be true + end + + it 'return false when rss exceed soft_limit_rss longer than GRACE_BALLOON_SECONDS' do + expect(memory_killer).to receive(:get_rss).exactly(4).times.and_return(250) + expect(memory_killer).to receive(:soft_limit_rss).exactly(5).times.and_return(200) + expect(memory_killer).to receive(:hard_limit_rss).exactly(5).times.and_return(300) + + expect(Time).to receive(:now).exactly(5).times.and_call_original + expect(memory_killer).to receive(:sleep).exactly(3).times.with(check_interval_seconds).and_call_original + + expect(memory_killer).to receive(:log_rss_out_of_range).with(250, 300, 200) + + expect(subject).to be false + end + end + + describe '#restart_sidekiq' do + let(:shutdown_timeout_seconds) { 7 } + + subject { memory_killer.send(:restart_sidekiq) } + + before do + stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) + allow(Sidekiq).to receive(:options).and_return(timeout: 9) + end + + it 'send signal' do + expect(memory_killer).to receive(:signal_and_wait).with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs').ordered + expect(memory_killer).to receive(:signal_and_wait).with(11, 'SIGTERM', 'gracefully shut down').ordered + expect(memory_killer).to receive(:signal_pgroup).with('SIGKILL', 'die').ordered + + subject + end + end + + describe '#signal_and_wait' do + let(:time) { 7 } + let(:signal) { 'my-signal' } + let(:explanation) { 'my-explanation' } + let(:check_interval_seconds) { 2 } + + subject { memory_killer.send(:signal_and_wait, time, signal, explanation) } + + before do + stub_const("#{described_class}::CHECK_INTERVAL_SECONDS", check_interval_seconds) + end + + it 'send signal and return when all jobs finished' do + expect(Process).to receive(:kill).with(signal, pid).ordered + expect(Time).to receive(:now).and_call_original + + expect(memory_killer).to receive(:enabled?).and_return(true) + expect(memory_killer).to receive(:any_jobs?).and_return(false) + + expect(memory_killer).not_to receive(:sleep) + + subject + end + + it 'send signal and wait till deadline if any job not finished' do + expect(Process).to receive(:kill).with(signal, pid).ordered + expect(Time).to receive(:now).and_call_original.at_least(:once) + + expect(memory_killer).to receive(:enabled?).and_return(true).at_least(:once) + expect(memory_killer).to receive(:any_jobs?).and_return(true).at_least(:once) + + expect(memory_killer).to receive(:sleep).and_call_original.exactly(4).times + + subject + end + end + + describe '#signal_pgroup' do + let(:signal) { 'my-signal' } + let(:explanation) { 'my-explanation' } + + subject { memory_killer.send(:signal_pgroup, signal, explanation) } + + it 'send signal to this proces if it is not group leader' do + expect(Process).to receive(:getpgrp).and_return(pid + 1) + + expect(Sidekiq.logger).to receive(:warn).once + .with( + class: described_class.to_s, + signal: signal, + pid: pid, + message: "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})") + expect(Process).to receive(:kill).with(signal, pid).ordered + + subject + end + + it 'send signal to whole process group as group leader' do + expect(Process).to receive(:getpgrp).and_return(pid) + + expect(Sidekiq.logger).to receive(:warn).once + .with( + class: described_class.to_s, + signal: signal, + pid: pid, + message: "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})") + expect(Process).to receive(:kill).with(signal, 0).ordered + + subject + end + end + + describe '#log_rss_out_of_range' do + let(:current_rss) { 100 } + let(:soft_limit_rss) { 200 } + let(:hard_limit_rss) { 300 } + let(:reason) { 'rss out of range reason description' } + + subject { memory_killer.send(:log_rss_out_of_range, current_rss, hard_limit_rss, soft_limit_rss) } + + it 'invoke sidekiq logger warn' do + expect(memory_killer).to receive(:out_of_range_description).with(current_rss, hard_limit_rss, soft_limit_rss).and_return(reason) + expect(Sidekiq.logger).to receive(:warn) + .with( + class: described_class.to_s, + pid: pid, + message: 'Sidekiq worker RSS out of range', + current_rss: current_rss, + hard_limit_rss: hard_limit_rss, + soft_limit_rss: soft_limit_rss, + reason: reason) + + subject + end + end + + describe '#out_of_range_description' do + let(:hard_limit) { 300 } + let(:soft_limit) { 200 } + let(:grace_balloon_seconds) { 12 } + + subject { memory_killer.send(:out_of_range_description, rss, hard_limit, soft_limit) } + + context 'when rss > hard_limit' do + let(:rss) { 400 } + + it 'tells reason' do + expect(subject).to eq("current_rss(#{rss}) > hard_limit_rss(#{hard_limit})") + end + end + + context 'when rss <= hard_limit' do + let(:rss) { 300 } + + it 'tells reason' do + stub_const("#{described_class}::GRACE_BALLOON_SECONDS", grace_balloon_seconds) + expect(subject).to eq("current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{grace_balloon_seconds})") + end + end + end + + describe '#rss_increase_by_jobs' do + let(:running_jobs) { { id1: 'job1', id2: 'job2' } } + + subject { memory_killer.send(:rss_increase_by_jobs) } + + it 'adds up individual rss_increase_by_job' do + expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return(running_jobs) + expect(memory_killer).to receive(:rss_increase_by_job).and_return(11, 22) + expect(subject).to eq(33) + end + + it 'return 0 if no job' do + expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return({}) + expect(subject).to eq(0) + end + end + + describe '#rss_increase_by_job' do + let(:worker_class) { Chaos::SleepWorker } + let(:job) { { worker_class: worker_class, started_at: 321 } } + let(:max_memory_kb) { 100000 } + + subject { memory_killer.send(:rss_increase_by_job, job) } + + before do + stub_const("#{described_class}::MAX_MEMORY_KB", max_memory_kb) + end + + it 'return 0 if memory_growth_kb return 0' do + expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_memory_growth_kb', 0).and_return(0) + expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_max_memory_growth_kb', max_memory_kb).and_return(0) + + expect(Time).not_to receive(:now) + expect(subject).to eq(0) + end + + it 'return time factored growth value when it does not exceed max growth limit for whilited job' do + expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_memory_growth_kb', 0).and_return(10) + expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_max_memory_growth_kb', max_memory_kb).and_return(100) + + expect(Time).to receive(:now).and_return(323) + expect(subject).to eq(20) + end + + it 'return max growth limit when time factored growth value exceed max growth limit for whilited job' do + expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_memory_growth_kb', 0).and_return(10) + expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_max_memory_growth_kb', max_memory_kb).and_return(100) + + expect(Time).to receive(:now).and_return(332) + expect(subject).to eq(100) + end + end + + describe '#get_job_options' do + let(:worker_class) { Chaos::SleepWorker } + let(:job) { { worker_class: worker_class, started_at: 321 } } + let(:key) { 'my-key' } + let(:default) { 'my-default' } + + subject { memory_killer.send(:get_job_options, job, key, default) } + + it 'return default if key is not defined' do + expect(worker_class).to receive(:sidekiq_options).and_return({ "retry" => 5 }) + + expect(subject).to eq(default) + end + + it 'return default if get StandardError when retrieve sidekiq_options' do + expect(worker_class).to receive(:sidekiq_options).and_raise(StandardError) + + expect(subject).to eq(default) + end + + it 'return right value if sidekiq_options has the key' do + expect(worker_class).to receive(:sidekiq_options).and_return({ key => 10 }) + + expect(subject).to eq(10) + end + end +end diff --git a/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb b/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb index acbb09e3542..397098ed5a4 100644 --- a/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb @@ -8,12 +8,12 @@ describe Gitlab::SidekiqDaemon::Monitor do describe '#within_job' do it 'tracks thread' do blk = proc do - expect(monitor.jobs_thread['jid']).not_to be_nil + expect(monitor.jobs.dig('jid', :thread)).not_to be_nil "OK" end - expect(monitor.within_job('jid', 'queue', &blk)).to eq("OK") + expect(monitor.within_job('worker_class', 'jid', 'queue', &blk)).to eq("OK") end context 'when job is canceled' do @@ -25,19 +25,34 @@ describe Gitlab::SidekiqDaemon::Monitor do it 'does not execute a block' do expect do |blk| - monitor.within_job(jid, 'queue', &blk) + monitor.within_job('worker_class', jid, 'queue', &blk) rescue described_class::CancelledError end.not_to yield_control end it 'raises exception' do - expect { monitor.within_job(jid, 'queue') }.to raise_error( + expect { monitor.within_job('worker_class', jid, 'queue') }.to raise_error( described_class::CancelledError) end end end - describe '#start_working' do + describe '#start_working when notification channel not enabled' do + subject { monitor.send(:start_working) } + + it 'return directly' do + allow(monitor).to receive(:notification_channel_enabled?).and_return(nil) + + expect(Sidekiq.logger).not_to receive(:info) + expect(Sidekiq.logger).not_to receive(:warn) + expect(monitor).not_to receive(:enabled?) + expect(monitor).not_to receive(:process_messages) + + subject + end + end + + describe '#start_working when notification channel enabled' do subject { monitor.send(:start_working) } before do @@ -45,6 +60,7 @@ describe Gitlab::SidekiqDaemon::Monitor do # we toggle `enabled?` flag after the first call stub_const('Gitlab::SidekiqDaemon::Monitor::RECONNECT_TIME', 0) allow(monitor).to receive(:enabled?).and_return(true, false) + allow(monitor).to receive(:notification_channel_enabled?).and_return(1) allow(Sidekiq.logger).to receive(:info) allow(Sidekiq.logger).to receive(:warn) @@ -204,7 +220,7 @@ describe Gitlab::SidekiqDaemon::Monitor do let(:thread) { Thread.new { sleep 1000 } } before do - monitor.jobs_thread[jid] = thread + monitor.jobs[jid] = { worker_class: 'worker_class', thread: thread, started_at: Time.now.to_i } end after do @@ -258,4 +274,24 @@ describe Gitlab::SidekiqDaemon::Monitor do subject end end + + describe '#notification_channel_enabled?' do + subject { monitor.send(:notification_channel_enabled?) } + + it 'return nil when SIDEKIQ_MONITOR_WORKER is not set' do + expect(subject).to be nil + end + + it 'return nil when SIDEKIQ_MONITOR_WORKER set to 0' do + allow(ENV).to receive(:fetch).with('SIDEKIQ_MONITOR_WORKER', 0).and_return("0") + + expect(subject).to be nil + end + + it 'return 1 when SIDEKIQ_MONITOR_WORKER set to 1' do + allow(ENV).to receive(:fetch).with('SIDEKIQ_MONITOR_WORKER', 0).and_return("1") + + expect(subject).to be 1 + end + end end diff --git a/spec/lib/gitlab/sidekiq_middleware/monitor_spec.rb b/spec/lib/gitlab/sidekiq_middleware/monitor_spec.rb index 023df1a6391..398144025ea 100644 --- a/spec/lib/gitlab/sidekiq_middleware/monitor_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/monitor_spec.rb @@ -12,7 +12,7 @@ describe Gitlab::SidekiqMiddleware::Monitor do it 'calls Gitlab::SidekiqDaemon::Monitor' do expect(Gitlab::SidekiqDaemon::Monitor.instance).to receive(:within_job) - .with('job-id', 'my-queue') + .with(anything, 'job-id', 'my-queue') .and_call_original expect { |blk| monitor.call(worker, job, queue, &blk) }.to yield_control diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 62787c5abaf..842dc4d511c 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -55,6 +55,7 @@ describe Gitlab::UsageData do omniauth_enabled reply_by_email_enabled container_registry_enabled + dependency_proxy_enabled gitlab_shared_runners_enabled gitlab_pages git @@ -234,6 +235,7 @@ describe Gitlab::UsageData do expect(subject[:omniauth_enabled]).to eq(Gitlab::Auth.omniauth_enabled?) expect(subject[:reply_by_email_enabled]).to eq(Gitlab::IncomingEmail.enabled?) expect(subject[:container_registry_enabled]).to eq(Gitlab.config.registry.enabled) + expect(subject[:dependency_proxy_enabled]).to eq(Gitlab.config.dependency_proxy.enabled) expect(subject[:gitlab_shared_runners_enabled]).to eq(Gitlab.config.gitlab_ci.shared_runners_enabled) end end |