Add latest changes from gitlab-org/gitlab@master

5 files changed, 264 insertions, 0 deletions

diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/client_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/client_spec.rb
new file mode 100644
index 00000000000..b6e47afc7e8
--- /dev/null
+++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/client_spec.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::SidekiqMiddleware::DuplicateJobs::Client, :clean_gitlab_redis_queues do
+  let(:worker_class) do
+    Class.new do
+      def self.name
+        'TestDeduplicationWorker'
+      end
+
+      include ApplicationWorker
+
+      def perform(*args)
+      end
+    end
+  end
+
+  before do
+    stub_const('TestDeduplicationWorker', worker_class)
+  end
+
+  describe '#call' do
+    it 'adds a correct duplicate tag to the jobs', :aggregate_failures do
+      TestDeduplicationWorker.bulk_perform_async([['args1'], ['args2'], ['args1']])
+
+      job1, job2, job3 = TestDeduplicationWorker.jobs
+
+      expect(job1['duplicate-of']).to be_nil
+      expect(job2['duplicate-of']).to be_nil
+      expect(job3['duplicate-of']).to eq(job1['jid'])
+    end
+  end
+end
diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb
new file mode 100644
index 00000000000..2334439461e
--- /dev/null
+++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb
@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gitlab_redis_queues do
+  subject(:duplicate_job) do
+    described_class.new(job, queue)
+  end
+
+  let(:job) { { 'class' => 'AuthorizedProjectsWorker', 'args' => [1], 'jid' => '123' } }
+  let(:queue) { 'authorized_projects' }
+
+  let(:idempotency_key) do
+    hash = Digest::SHA256.hexdigest("#{job['class']}:#{job['args'].join('-')}")
+    "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:duplicate:#{queue}:#{hash}"
+  end
+
+  describe '#schedule' do
+    it 'calls schedule on the strategy' do
+      expect do |block|
+        expect_next_instance_of(Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuting) do |strategy|
+          expect(strategy).to receive(:schedule).with(job, &block)
+        end
+
+        duplicate_job.schedule(&block)
+      end.to yield_control
+    end
+  end
+
+  describe '#perform' do
+    it 'calls perform on the strategy' do
+      expect do |block|
+        expect_next_instance_of(Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuting) do |strategy|
+          expect(strategy).to receive(:perform).with(job, &block)
+        end
+
+        duplicate_job.perform(&block)
+      end.to yield_control
+    end
+  end
+
+  describe '#check!' do
+    context 'when there was no job in the queue yet' do
+      it { expect(duplicate_job.check!).to eq('123') }
+
+      it "adds a key with ttl set to #{described_class::DUPLICATE_KEY_TTL}" do
+        expect { duplicate_job.check! }
+          .to change { read_idempotency_key_with_ttl(idempotency_key) }
+                .from([nil, -2])
+                .to(['123', be_within(1).of(described_class::DUPLICATE_KEY_TTL)])
+      end
+    end
+
+    context 'when there was already a job with same arguments in the same queue' do
+      before do
+        set_idempotency_key(idempotency_key, 'existing-key')
+      end
+
+      it { expect(duplicate_job.check!).to eq('existing-key') }
+
+      it "does not change the existing key's TTL" do
+        expect { duplicate_job.check! }
+          .not_to change { read_idempotency_key_with_ttl(idempotency_key) }
+                .from(['existing-key', -1])
+      end
+
+      it 'sets the existing jid' do
+        duplicate_job.check!
+
+        expect(duplicate_job.existing_jid).to eq('existing-key')
+      end
+    end
+  end
+
+  describe '#delete!' do
+    context "when we didn't track the definition" do
+      it { expect { duplicate_job.delete! }.not_to raise_error }
+    end
+
+    context 'when the key exists in redis' do
+      before do
+        set_idempotency_key(idempotency_key, 'existing-key')
+      end
+
+      it 'removes the key from redis' do
+        expect { duplicate_job.delete! }
+          .to change { read_idempotency_key_with_ttl(idempotency_key) }
+                .from(['existing-key', -1])
+                .to([nil, -2])
+      end
+    end
+  end
+
+  describe '#duplicate?' do
+    it "raises an error if the check wasn't performed" do
+      expect { duplicate_job.duplicate? }.to raise_error /Call `#check!` first/
+    end
+
+    it 'returns false if the existing jid equals the job jid' do
+      duplicate_job.check!
+
+      expect(duplicate_job.duplicate?).to be(false)
+    end
+
+    it 'returns false if the existing jid is different from the job jid' do
+      set_idempotency_key(idempotency_key, 'a different jid')
+      duplicate_job.check!
+
+      expect(duplicate_job.duplicate?).to be(true)
+    end
+  end
+
+  def set_idempotency_key(key, value = '1')
+    Sidekiq.redis { |r| r.set(key, value) }
+  end
+
+  def read_idempotency_key_with_ttl(key)
+    Sidekiq.redis do |redis|
+      redis.pipelined do |p|
+        p.get(key)
+        p.ttl(key)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb
new file mode 100644
index 00000000000..0ea248fbcf1
--- /dev/null
+++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_redis_queues do
+  let(:worker_class) do
+    Class.new do
+      def self.name
+        'TestDeduplicationWorker'
+      end
+
+      include ApplicationWorker
+
+      def perform(*args)
+      end
+    end
+  end
+
+  before do
+    stub_const('TestDeduplicationWorker', worker_class)
+  end
+
+  around do |example|
+    Sidekiq::Testing.inline! { example.run }
+  end
+
+  before(:context) do
+    Sidekiq::Testing.server_middleware do |chain|
+      chain.add described_class
+    end
+  end
+
+  after(:context) do
+    Sidekiq::Testing.server_middleware do |chain|
+      chain.remove described_class
+    end
+  end
+
+  describe '#call' do
+    it 'removes the stored job from redis' do
+      bare_job = { 'class' => 'TestDeduplicationWorker', 'args' => ['hello'] }
+      job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'test_deduplication')
+
+      expect(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob)
+        .to receive(:new).with(a_hash_including(bare_job), 'test_deduplication')
+              .and_return(job_definition).twice # once in client middleware
+      expect(job_definition).to receive(:delete!).and_call_original
+
+      TestDeduplicationWorker.perform_async('hello')
+    end
+  end
+end
diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing_spec.rb
new file mode 100644
index 00000000000..f40e829f9a5
--- /dev/null
+++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+describe Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuting do
+  let(:fake_duplicate_job) do
+    instance_double(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob)
+  end
+
+  subject(:strategy) { described_class.new(fake_duplicate_job) }
+
+  describe '#schedule' do
+    it 'checks for duplicates before yielding' do
+      expect(fake_duplicate_job).to receive(:check!).ordered.and_return('a jid')
+      expect(fake_duplicate_job).to receive(:duplicate?).ordered.and_return(false)
+      expect { |b| strategy.schedule({}, &b) }.to yield_control
+    end
+
+    it 'adds the jid of the existing job to the job hash' do
+      allow(fake_duplicate_job).to receive(:check!).and_return('the jid')
+      job_hash = {}
+
+      expect(fake_duplicate_job).to receive(:duplicate?).and_return(true)
+      expect(fake_duplicate_job).to receive(:existing_jid).and_return('the jid')
+
+      strategy.schedule(job_hash) {}
+
+      expect(job_hash).to include('duplicate-of' => 'the jid')
+    end
+  end
+
+  describe '#perform' do
+    it 'deletes the lock before executing' do
+      expect(fake_duplicate_job).to receive(:delete!).ordered
+      expect { |b| strategy.perform({}, &b) }.to yield_control
+    end
+  end
+end
diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies_spec.rb
new file mode 100644
index 00000000000..6ecc2a3a5f8
--- /dev/null
+++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies_spec.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+describe Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies do
+  describe '.for' do
+    it 'returns the right class for `until_executing`' do
+      expect(described_class.for(:until_executing)).to eq(described_class::UntilExecuting)
+    end
+
+    it 'raises an UnknownStrategyError when passing an unknown key' do
+      expect { described_class.for(:unknown) }.to raise_error(described_class::UnknownStrategyError)
+    end
+  end
+end